From ad8f3dfde70fdab47f3260f6f0ebcfd74fc8cc79 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 17 Mar 2025 14:55:40 -0400 Subject: [PATCH 01/31] use specified role on new user add --- salt/soc/files/bin/salt-relay.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index 18ce8c0b0..c9e3ba8c8 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -89,7 +89,7 @@ function manage_user() { add) email=$(echo "$request" | jq -r .email) password=$(echo "$request" | jq -r .password) - perm=$(echo "$request" | jq -r .role) + role=$(echo "$request" | jq -r .role) firstName=$(echo "$request" | jq -r .firstName) lastName=$(echo "$request" | jq -r .lastName) note=$(echo "$request" | jq -r .note) From af6245f19dc64b13ca5be0be52747546e9aa770a Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:30:17 -0500 Subject: [PATCH 02/31] add zeek file_extraction forcedType for instances where a single line is speciifed --- salt/zeek/soc_zeek.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index f5f718114..b3b655083 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -63,4 +63,5 @@ zeek: duplicates: True file_extraction: description: Contains a list of file or MIME types Zeek will extract from the network streams. Values must adhere to the following format - {"MIME_TYPE":"FILE_EXTENSION"} + forcedType: "[]{}" helpLink: zeek.html From 4d7fdd390c639f825ff0b3417ec9cc936da569e7 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 18 Mar 2025 08:52:43 -0500 Subject: [PATCH 03/31] ldap_search include observer.name --- .../files/ingest/zeek.ldap_search | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/files/ingest/zeek.ldap_search b/salt/elasticsearch/files/ingest/zeek.ldap_search index 2a625c319..9bddf41b9 100644 --- a/salt/elasticsearch/files/ingest/zeek.ldap_search +++ b/salt/elasticsearch/files/ingest/zeek.ldap_search @@ -1,9 +1,25 @@ { "description":"zeek.ldap_search", "processors":[ - {"pipeline": {"name": "zeek.ldap", "ignore_missing_pipeline":true,"ignore_failure":true}}, - {"set": {"field": "event.dataset", "value":"ldap_search"}}, - {"remove": {"field": "tags", "ignore_missing":true}}, + {"set": {"field": "event.dataset", "value":"ldap_search"}}, + {"json": {"field": "message", "target_field": "message2", "ignore_failure": true}}, + {"rename": {"field": "message2.message_id", "target_field": "ldap.message_id", "ignore_missing": true}}, + {"rename": {"field": "message2.opcode", "target_field": "ldap.opcode", "ignore_missing": true}}, + {"rename": {"field": "message2.result", "target_field": "ldap.result", "ignore_missing": true}}, + {"rename": {"field": "message2.diagnostic_message", "target_field": "ldap.diagnostic_message", "ignore_missing": true}}, + {"rename": {"field": "message2.version", "target_field": "ldap.version", "ignore_missing": true}}, + {"rename": {"field": "message2.object", "target_field": "ldap.object", "ignore_missing": true}}, + {"rename": {"field": "message2.argument", "target_field": "ldap.argument", "ignore_missing": true}}, + {"rename": {"field": "message2.scope", "target_field": "ldap_search.scope", "ignore_missing":true}}, + {"rename": {"field": "message2.deref_aliases", "target_field": "ldap_search.deref_aliases", "ignore_missing":true}}, + {"rename": {"field": "message2.base_object", "target_field": "ldap.object", "ignore_missing":true}}, + {"rename": {"field": "message2.result_count", "target_field": "ldap_search.result_count", "ignore_missing":true}}, + {"rename": {"field": "message2.filter", "target_field": "ldap_search.filter", "ignore_missing":true}}, + {"rename": {"field": "message2.attributes", "target_field": "ldap_search.attributes", "ignore_missing":true}}, + {"script": {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('diagnostic_message') && ctx.ldap.diagnostic_message != null) {\n String message = ctx.ldap.diagnostic_message;\n\n // get user and property from SASL success\n if (message.toLowerCase().contains(\"sasl(0): successful result\")) {\n Pattern pattern = /user:\\s*([^ ]+)\\s*property:\\s*([^ ]+)/i;\n Matcher matcher = pattern.matcher(message);\n if (matcher.find()) {\n ctx.ldap.user_email = matcher.group(1); // Extract user email\n ctx.ldap.property = matcher.group(2); // Extract property\n }\n }\n if (message.toLowerCase().contains(\"ldaperr:\")) {\n Pattern pattern = /comment:\\s*([^,]+)/i;\n Matcher matcher = pattern.matcher(message);\n\n if (matcher.find()) {\n ctx.ldap.comment = matcher.group(1);\n }\n }\n }","ignore_failure": true}}, + {"script": {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('object') && ctx.ldap.object != null) {\n String message = ctx.ldap.object;\n\n // parse common name from ldap object\n if (message.toLowerCase().contains(\"cn=\")) {\n Pattern pattern = /cn=([^,]+)/i;\n Matcher matcher = pattern.matcher(message);\n if (matcher.find()) {\n ctx.ldap.common_name = matcher.group(1); // Extract CN\n }\n }\n // build domain from ldap object\n if (message.toLowerCase().contains(\"dc=\")) {\n Pattern dcPattern = /dc=([^,]+)/i;\n Matcher dcMatcher = dcPattern.matcher(message);\n\n StringBuilder domainBuilder = new StringBuilder();\n while (dcMatcher.find()) {\n if (domainBuilder.length() > 0 ){\n domainBuilder.append(\".\");\n }\n domainBuilder.append(dcMatcher.group(1));\n }\n if (domainBuilder.length() > 0) {\n ctx.ldap.domain = domainBuilder.toString();\n }\n }\n // create list of any organizational units from ldap object\n if (message.toLowerCase().contains(\"ou=\")) {\n Pattern ouPattern = /ou=([^,]+)/i;\n Matcher ouMatcher = ouPattern.matcher(message);\n ctx.ldap.organizational_unit = [];\n\n while (ouMatcher.find()) {\n ctx.ldap.organizational_unit.add(ouMatcher.group(1));\n }\n if(ctx.ldap.organizational_unit.isEmpty()) {\n ctx.remove(\"ldap.organizational_unit\");\n }\n }\n}\n","ignore_failure": true}}, + {"remove": {"field": "message2.tags", "ignore_failure": true}}, + {"remove": {"field": ["host"], "ignore_failure": true}}, {"pipeline": {"name": "zeek.common"}} ] } \ No newline at end of file From 3f3e7ea1e84444a40ad764ab28dc1e8b262806fb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 18 Mar 2025 10:12:23 -0400 Subject: [PATCH 04/31] add no-op soup functions for 2.4.140 --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3d5ca338f..ff5331584 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -407,6 +407,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.111 [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120 [[ "$INSTALLEDVERSION" == 2.4.120 ]] && up_to_2.4.130 + [[ "$INSTALLEDVERSION" == 2.4.130 ]] && up_to_2.4.140 true } @@ -431,6 +432,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.110 ]] && post_to_2.4.111 [[ "$POSTVERSION" == 2.4.111 ]] && post_to_2.4.120 [[ "$POSTVERSION" == 2.4.120 ]] && post_to_2.4.130 + [[ "$POSTVERSION" == 2.4.130 ]] && post_to_2.4.140 true } @@ -553,6 +555,11 @@ post_to_2.4.130() { POSTVERSION=2.4.130 } +post_to_2.4.140() { + echo "Nothing to apply" + POSTVERSION=2.4.140 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -773,6 +780,12 @@ up_to_2.4.130() { INSTALLEDVERSION=2.4.130 } +up_to_2.4.140() { + echo "Nothing to do for 2.4.140" + + INSTALLEDVERSION=2.4.140 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls From 1b98f9f3137c84eec2e3204062f43e9bb682d986 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Mar 2025 10:03:26 -0400 Subject: [PATCH 05/31] Update bootstrap-salt.sh --- salt/salt/scripts/bootstrap-salt.sh | 973 ++++++++++++++++++++-------- 1 file changed, 719 insertions(+), 254 deletions(-) diff --git a/salt/salt/scripts/bootstrap-salt.sh b/salt/salt/scripts/bootstrap-salt.sh index 4e65008dc..8f2956606 100644 --- a/salt/salt/scripts/bootstrap-salt.sh +++ b/salt/salt/scripts/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.07" +__ScriptVersion="2025.02.24" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -221,6 +221,7 @@ __check_config_dir() { # DESCRIPTION: Checks the placed after the install arguments #---------------------------------------------------------------------------------------------------------------------- __check_unparsed_options() { + shellopts="$1" # grep alternative for SunOS if [ -f /usr/xpg4/bin/grep ]; then @@ -543,8 +544,8 @@ __exit_cleanup() { echodebug "Cleaning up the Salt Temporary Git Repository" # shellcheck disable=SC2164 cd "${__SALT_GIT_CHECKOUT_PARENT_DIR}" - rm -rf "${_SALT_GIT_CHECKOUT_DIR}" - #rm -rf "${_SALT_GIT_CHECKOUT_DIR}/deps" + rm -fR "${_SALT_GIT_CHECKOUT_DIR}" + #rm -fR "${_SALT_GIT_CHECKOUT_DIR}/deps" else echowarn "Not cleaning up the Salt Temporary git repository on request" echowarn "Note that if you intend to re-run this script using the git approach, you might encounter some issues" @@ -618,13 +619,24 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git|onedir|onedir_rc)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(latest|default|stable|testing|git|onedir|onedir_rc)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi +## allows GitHub Actions CI/CD easier handling of latest and default +if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then + STABLE_REV="latest" + ONEDIR_REV="latest" + _ONEDIR_REV="latest" + ITYPE="onedir" + if [ "$#" -gt 0 ];then + shift + fi + echodebug "using ITYPE onedir for input 'latest' or 'default', cmd args left ,$#," + # If doing a git install, check what branch/tag/sha will be checked out -if [ "$ITYPE" = "git" ]; then +elif [ "$ITYPE" = "git" ]; then if [ "$#" -eq 0 ];then GIT_REV="master" else @@ -649,7 +661,7 @@ elif [ "$ITYPE" = "stable" ]; then _ONEDIR_REV="$1" ITYPE="onedir" shift - elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then STABLE_REV="$1" ONEDIR_REV="$1" _ONEDIR_REV="$1" @@ -849,7 +861,7 @@ if [ "$ITYPE" != "git" ]; then fi fi -# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com +# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com/artifactory if [ "$_CUSTOM_REPO_URL" != "null" ]; then _REPO_URL="$_CUSTOM_REPO_URL" @@ -920,6 +932,7 @@ fi # DESCRIPTION: Retrieves a URL and writes it to a given path #---------------------------------------------------------------------------------------------------------------------- __fetch_url() { + # shellcheck disable=SC2086 curl $_CURL_ARGS -L -s -f -o "$1" "$2" >/dev/null 2>&1 || wget $_WGET_ARGS -q -O "$1" "$2" >/dev/null 2>&1 || @@ -1924,6 +1937,7 @@ __function_defined() { # process is finished so the script doesn't exit on a locked proc. #---------------------------------------------------------------------------------------------------------------------- __wait_for_apt(){ + # Timeout set at 15 minutes WAIT_TIMEOUT=900 @@ -2003,7 +2017,6 @@ __temp_gpg_pub() { #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - url=$1 tempfile="$(__temp_gpg_pub)" @@ -2080,6 +2093,7 @@ __tdnf_install_noinput() { # DESCRIPTION: (DRY) Helper function to clone and checkout salt to a # specific revision. #---------------------------------------------------------------------------------------------------------------------- +# shellcheck disable=SC2120 __git_clone_and_checkout() { echodebug "Installed git version: $(git --version | awk '{ print $3 }')" @@ -2088,6 +2102,12 @@ __git_clone_and_checkout() { export GIT_SSL_NO_VERIFY=1 fi + if [ "$(echo "$GIT_REV" | grep -E '^(3006|3007)$')" != "" ]; then + GIT_REV_ADJ="$GIT_REV.x" # branches are 3006.x or 3007.x + else + GIT_REV_ADJ="$GIT_REV" + fi + __SALT_GIT_CHECKOUT_PARENT_DIR=$(dirname "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null) __SALT_GIT_CHECKOUT_PARENT_DIR="${__SALT_GIT_CHECKOUT_PARENT_DIR:-/tmp/git}" __SALT_CHECKOUT_REPONAME="$(basename "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null)" @@ -2116,15 +2136,15 @@ __git_clone_and_checkout() { git fetch --tags upstream fi - echodebug "Hard reseting the cloned repository to ${GIT_REV}" - git reset --hard "$GIT_REV" || return 1 + echodebug "Hard reseting the cloned repository to ${GIT_REV_ADJ}" + git reset --hard "$GIT_REV_ADJ" || return 1 - # Just calling `git reset --hard $GIT_REV` on a branch name that has + # Just calling `git reset --hard $GIT_REV_ADJ` on a branch name that has # already been checked out will not update that branch to the upstream # HEAD; instead it will simply reset to itself. Check the ref to see # if it is a branch name, check out the branch, and pull in the # changes. - if git branch -a | grep -q "${GIT_REV}"; then + if git branch -a | grep -q "${GIT_REV_ADJ}"; then echodebug "Rebasing the cloned repository branch" git pull --rebase || return 1 fi @@ -2141,21 +2161,19 @@ __git_clone_and_checkout() { fi if [ "$__SHALLOW_CLONE" -eq $BS_TRUE ]; then - # Let's try shallow cloning to speed up. - # Test for "--single-branch" option introduced in git 1.7.10, the minimal version of git where the shallow + # Let's try 'treeless' cloning to speed up. Treeless cloning omits trees and blobs ('files') + # but includes metadata (commit history, tags, branches etc. + # Test for "--filter" option introduced in git 2.19, the minimal version of git where the treeless # cloning we need actually works - if [ "$(git clone 2>&1 | grep 'single-branch')" != "" ]; then - # The "--single-branch" option is supported, attempt shallow cloning - echoinfo "Attempting to shallow clone $GIT_REV from Salt's repository ${_SALT_REPO_URL}" - ## Shallow cloning is resulting in the wrong version of Salt, even with a depth of 5 - ## getting 3007.0+0na.246d066 when it should be 3007.1+410.g246d066457, disabling for now - ## if git clone --depth 1 --branch "$GIT_REV" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then - echodebug "git command, git clone --branch $GIT_REV $_SALT_REPO_URL $__SALT_CHECKOUT_REPONAME" - if git clone --branch "$GIT_REV" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then + if [ "$(git clone 2>&1 | grep 'filter')" != "" ]; then + # The "--filter" option is supported: attempt treeless cloning + echoinfo "Attempting to shallow clone $GIT_REV_ADJ from Salt's repository ${_SALT_REPO_URL}" + echodebug "git command, git clone --filter=tree:0 --branch $GIT_REV_ADJ $_SALT_REPO_URL $__SALT_CHECKOUT_REPONAME" + if git clone --filter=tree:0 --branch "$GIT_REV_ADJ" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then # shellcheck disable=SC2164 cd "${_SALT_GIT_CHECKOUT_DIR}" __SHALLOW_CLONE=$BS_TRUE - echoinfo "shallow path (disabled shallow) git cloned $GIT_REV, version $(python3 salt/version.py)" + echoinfo "shallow path git cloned $GIT_REV_ADJ, version $(python3 salt/version.py)" else # Shallow clone above failed(missing upstream tags???), let's resume the old behaviour. echowarn "Failed to shallow clone." @@ -2174,7 +2192,7 @@ __git_clone_and_checkout() { # shellcheck disable=SC2164 cd "${_SALT_GIT_CHECKOUT_DIR}" - echoinfo "git cloned $GIT_REV, version $(python3 salt/version.py)" + echoinfo "git cloned $GIT_REV_ADJ, version $(python3 salt/version.py)" if ! echo "$_SALT_REPO_URL" | grep -q -F -w "${_SALTSTACK_REPO_URL#*://}"; then # We need to add the saltstack repository as a remote and fetch tags for proper versioning @@ -2184,14 +2202,14 @@ __git_clone_and_checkout() { echodebug "Fetching upstream (SaltStack's Salt repository) git tags" git fetch --tags upstream || return 1 - # Check if GIT_REV is a remote branch or just a commit hash - if git branch -r | grep -q -F -w "origin/$GIT_REV"; then - GIT_REV="origin/$GIT_REV" + # Check if GIT_REV_ADJ is a remote branch or just a commit hash + if git branch -r | grep -q -F -w "origin/$GIT_REV_ADJ"; then + GIT_REV_ADJ="origin/$GIT_REV_ADJ" fi fi - echodebug "Checking out $GIT_REV" - git checkout "$GIT_REV" || return 1 + echodebug "Checking out $GIT_REV_ADJ" + git checkout "$GIT_REV_ADJ" || return 1 fi fi @@ -2437,10 +2455,12 @@ __check_services_systemd() { _SYSTEMD_ACTIVE=$(/bin/systemctl daemon-reload 2>&1 | grep 'System has not been booted with systemd') echodebug "__check_services_systemd _SYSTEMD_ACTIVE result ,$_SYSTEMD_ACTIVE," - if [ "$_SYSTEMD_ACTIVE" != "" ]; then + if [ -n "$_SYSTEMD_ACTIVE" ]; then _SYSTEMD_FUNCTIONAL=$BS_FALSE echodebug "systemd is not functional, despite systemctl being present, setting _SYSTEMD_FUNCTIONAL false, $_SYSTEMD_FUNCTIONAL" return 1 + else + echodebug "systemd is functional, _SYSTEMD_FUNCTIONAL true, $_SYSTEMD_FUNCTIONAL" fi servicename=$1 @@ -2611,6 +2631,7 @@ __activate_virtualenv() { #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { + _pip_pkgs="$1" _py_exe="$2" _py_pkg=$(echo "$_py_exe" | sed -E "s/\\.//g") @@ -2632,8 +2653,10 @@ __install_pip_pkgs() { else __PACKAGES="${__PACKAGES} ${_py_pkg}-devel" if [ "$DISTRO_NAME_L" = "fedora" ];then + dnf makecache || return 1 __dnf_install_noinput ${__PACKAGES} || return 1 else + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 fi fi @@ -2652,6 +2675,7 @@ __install_pip_pkgs() { # PARAMETERS: requirements_file #---------------------------------------------------------------------------------------------------------------------- __install_pip_deps() { + # Install virtualenv to system pip before activating virtualenv if thats going to be used # We assume pip pkg is installed since that is distro specific if [ "$_VIRTUALENV_DIR" != "null" ]; then @@ -2716,44 +2740,6 @@ __install_salt_from_repo() { echodebug "Installed pip version: $(${_pip_cmd} --version)" - CHECK_PIP_VERSION_SCRIPT=$(cat << EOM -import sys -try: - import pip - installed_pip_version=tuple([int(part.strip()) for part in pip.__version__.split('.') if part.isdigit()]) - desired_pip_version=($(echo ${_MINIMUM_PIP_VERSION} | sed 's/\./, /g' )) - if installed_pip_version < desired_pip_version: - print('Desired pip version {!r} > Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) - sys.exit(1) - print('Desired pip version {!r} < Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) - sys.exit(0) -except ImportError: - print('Failed to import pip') - sys.exit(1) -EOM -) - if ! ${_py_exe} -c "$CHECK_PIP_VERSION_SCRIPT"; then - # Upgrade pip to at least 1.2 which is when we can start using "python3 -m pip" - echodebug "Running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} pip>=${_MINIMUM_PIP_VERSION}'" - ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v "pip>=${_MINIMUM_PIP_VERSION}" - sleep 1 - echodebug "PATH: ${PATH}" - _pip_cmd="pip${_py_version}" - if ! __check_command_exists "${_pip_cmd}"; then - echodebug "The pip binary '${_pip_cmd}' was not found in PATH" - _pip_cmd="pip$(echo "${_py_version}" | cut -c -1)" - if ! __check_command_exists "${_pip_cmd}"; then - echodebug "The pip binary '${_pip_cmd}' was not found in PATH" - _pip_cmd="pip" - if ! __check_command_exists "${_pip_cmd}"; then - echoerror "Unable to find a pip binary" - return 1 - fi - fi - fi - echodebug "Installed pip version: $(${_pip_cmd} --version)" - fi - _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION},<${_MAXIMUM_SETUPTOOLS_VERSION}" if [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -2776,25 +2762,37 @@ EOM mkdir -p /tmp/git/deps echodebug "Created directory /tmp/git/deps" - echodebug "Installing Salt dependencies for Salt version $(python3 salt/version.py)" if [ ${DISTRO_NAME_L} = "ubuntu" ] && [ "$DISTRO_MAJOR_VERSION" -eq 22 ]; then echodebug "Ubuntu 22.04 has problem with base.txt requirements file, not parsing sys_platform == 'win32', upgrading from default pip works" echodebug "${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade pip" - ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade pip || (echo "Failed to upgrade pip" && return 1) + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade pip + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "Failed to upgrade pip" + return 1 + fi fi - echoinfo "Downloading Salt Dependencies from PyPi" - echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" - ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . || (echo "Failed to download salt dependencies" && return 1) - - echoinfo "Installing Downloaded Salt Dependencies" - echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/*'" - ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 rm -f /tmp/git/deps/* - echoinfo "Building Salt Python Wheel" + echodebug "Installing Salt requirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "Failed to install salt requirements for the version of Python ${_py_version}" + return 1 + fi + if [ "${OS_NAME}" = "Linux" ]; then + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.functools==4.1.0" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.text==4.0.0" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.collections==5.1.0" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.context==6.0.1" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.classes==3.4.0" || return 1 + fi + + echoinfo "Building Salt Python Wheel" if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then SETUP_PY_INSTALL_ARGS="-v" fi @@ -2937,6 +2935,8 @@ fi # __enable_universe_repository() { + echodebug "__enable_universe_repository() entry" + if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled return 0 @@ -2950,6 +2950,7 @@ __enable_universe_repository() { } __install_saltstack_ubuntu_repository() { + # Workaround for latest non-LTS Ubuntu echodebug "__install_saltstack_ubuntu_repository() entry" @@ -2993,14 +2994,13 @@ __install_saltstack_ubuntu_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3010,6 +3010,7 @@ __install_saltstack_ubuntu_repository() { __install_saltstack_ubuntu_onedir_repository() { echodebug "__install_saltstack_ubuntu_onedir_repository() entry" + # Workaround for latest non-LTS Ubuntu if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ { [ "$DISTRO_MAJOR_VERSION" -eq 22 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ @@ -3031,7 +3032,7 @@ __install_saltstack_ubuntu_onedir_repository() { fi ## include hwclock if not part of base OS (23.10 and up) - if [ ! -f /usr/sbin/hwclock ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 23 ] && [ ! -f /usr/sbin/hwclock ]; then __PACKAGES="${__PACKAGES} util-linux-extra" fi @@ -3045,14 +3046,14 @@ __install_saltstack_ubuntu_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV_DOT" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3099,6 +3100,9 @@ install_ubuntu_deps() { # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES="${__PACKAGES} procps pciutils" + # ensure sudo, ps installed + __PACKAGES="${__PACKAGES} sudo" + ## include hwclock if not part of base OS (23.10 and up) if [ ! -f /usr/sbin/hwclock ]; then __PACKAGES="${__PACKAGES} util-linux-extra" @@ -3165,6 +3169,7 @@ install_ubuntu_git_deps() { __apt_get_install_noinput ca-certificates fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -3182,6 +3187,9 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi + # Additionally install procps pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3229,6 +3237,8 @@ install_ubuntu_onedir_deps() { install_ubuntu_stable() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3286,6 +3296,8 @@ install_ubuntu_git() { install_ubuntu_onedir() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3322,7 +3334,6 @@ install_ubuntu_stable_post() { [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Using systemd /bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( @@ -3480,25 +3491,21 @@ __install_saltstack_debian_repository() { # shellcheck disable=SC2086,SC2090 __apt_get_install_noinput ${__PACKAGES} || return 1 - ## SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" - ## echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - ## __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - ## __wait_for_apt apt-get update || return 1 - __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV_DOT" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3535,14 +3542,14 @@ __install_saltstack_debian_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV_DOT" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3580,8 +3587,8 @@ install_debian_onedir_deps() { return 1 fi - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES='procps pciutils' + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES='procps pciutils sudo' # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" @@ -3617,6 +3624,7 @@ install_debian_git_deps() { __apt_get_install_noinput ca-certificates fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -3627,6 +3635,9 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3641,6 +3652,8 @@ install_debian_git_deps() { install_debian_stable() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3720,6 +3733,8 @@ install_debian_12_git() { install_debian_onedir() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3862,12 +3877,28 @@ __install_saltstack_fedora_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -3875,6 +3906,7 @@ __install_saltstack_fedora_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." @@ -3904,7 +3936,7 @@ install_fedora_deps() { __PACKAGES="${__PACKAGES} dnf-utils libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd sudo" if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" fi @@ -3936,9 +3968,11 @@ install_fedora_git_deps() { __PACKAGES="" fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" + __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++ sudo" + # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 @@ -4066,7 +4100,7 @@ install_fedora_onedir_deps() { __install_saltstack_fedora_onedir_repository || return 1 fi - __PACKAGES="dnf-utils chkconfig procps-ng" + __PACKAGES="dnf-utils chkconfig procps-ng sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4086,27 +4120,38 @@ install_fedora_onedir() { STABLE_REV=$ONEDIR_REV #install_fedora_stable || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" + else + MINOR_VER_STRG="" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 + dnf makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 @@ -4153,19 +4198,36 @@ __install_saltstack_rhel_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-latest fi - dnf clean expire-cache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." @@ -4198,7 +4260,7 @@ install_centos_stable_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4214,26 +4276,38 @@ install_centos_stable_deps() { install_centos_stable() { + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 # Workaround for 3.11 broken on CentOS Stream 8.x @@ -4296,6 +4370,7 @@ install_centos_git_deps() { __yum_install_noinput git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __PACKAGES="" @@ -4309,7 +4384,7 @@ install_centos_git_deps() { return 1 fi - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4405,7 +4480,7 @@ install_centos_onedir_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4420,30 +4495,42 @@ install_centos_onedir_deps() { } install_centos_onedir() { - yum clean metadata - yum makecache + + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$ONEDIR_REV_DOT" + else + MINOR_VER_STRG="" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud-${STABLE_REV}" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master-${STABLE_REV}" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion-${STABLE_REV}" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic-${STABLE_REV}" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api-${STABLE_REV}" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 + yum makecache || return 1 + yum list salt-minion || return 1 __yum_install_noinput ${__PACKAGES} || return 1 + return 0 } @@ -4993,7 +5080,7 @@ install_oracle_linux_check_services() { ####################################################################################################################### # -# AlmaLinux Install Functions +# ALmaLinux Install Functions # install_almalinux_stable_deps() { install_centos_stable_deps || return 1 @@ -5324,6 +5411,7 @@ install_alpine_linux_git_deps() { apk -U add git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 apk -U add python3 python3-dev py3-pip py3-setuptools g++ linux-headers zeromq-dev openrc || return 1 @@ -5490,9 +5578,10 @@ install_amazon_linux_ami_2_git_deps() { __yum_install_noinput git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5514,7 +5603,9 @@ install_amazon_linux_ami_2_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5523,22 +5614,67 @@ install_amazon_linux_ami_2_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then if [ ! -s "${YUM_REPO_FILE}" ]; then - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + ## Amazon Linux yum (v3) doesn't support config-manager + ## FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + # shellcheck disable=SC2129 if [ "$STABLE_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + fi + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + echo "[salt-repo-${STABLE_REV_DOT}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${STABLE_REV_DOT} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi - dnf clean expire-cache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 fi fi @@ -5557,7 +5693,9 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5566,22 +5704,67 @@ install_amazon_linux_ami_2_onedir_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then if [ ! -s "${YUM_REPO_FILE}" ]; then - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + ## Amazon Linux yum (v3) doesn't support config-manager + ## FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi - dnf clean expire-cache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 fi fi @@ -5664,9 +5847,10 @@ install_amazon_linux_ami_2023_git_deps() { __yum_install_noinput git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5680,11 +5864,20 @@ install_amazon_linux_ami_2023_git_deps() { return 0 } +install_amazon_linux_ami_2023_deps() { + + # Set ONEDIR_REV to STABLE_REV + ONEDIR_REV=${STABLE_REV} + install_amazon_linux_ami_2023_onedir_deps || return 1 +} + install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5693,22 +5886,67 @@ install_amazon_linux_ami_2023_onedir_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then if [ ! -s "${YUM_REPO_FILE}" ]; then - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + ## Amazon Linux yum (v3) doesn't support config-manager + ## FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi - dnf clean expire-cache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 fi fi @@ -5827,6 +6065,7 @@ install_arch_linux_git_deps() { pacman -Sy --noconfirm --needed git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 2 ]; then @@ -6025,6 +6264,37 @@ install_arch_linux_onedir_post() { # Photon OS Install Functions # +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __rpm_get_packagesite_onedir_latest +# DESCRIPTION: Set _GENERIC_PKG_VERSION to the latest for RPM or latest for major version input +#---------------------------------------------------------------------------------------------------------------------- +__get_packagesite_onedir_latest() { + + echodebug "Find latest rpm release from repository" + + # get dir listing from url, sort and pick highest + generic_versions_tmpdir=$(mktemp -d) + curr_pwd=$(pwd) + cd ${generic_versions_tmpdir} || return 1 + + # leverage the windows directories since release Windows and Linux + wget -q -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" + if [ "$#" -gt 0 ] && [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi + cd ${curr_pwd} || return "${_GENERIC_PKG_VERSION}" + rm -fR ${generic_versions_tmpdir} + + echodebug "latest rpm release from repository found ${_GENERIC_PKG_VERSION}" + +} + + __install_saltstack_photon_onedir_repository() { echodebug "__install_saltstack_photon_onedir_repository() entry" @@ -6039,34 +6309,49 @@ __install_saltstack_photon_onedir_repository() { ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - ## tdnf config-manager --set-disable salt-repo-* - ## tdnf config-manager --set-enabled salt-repo-3007-sts - echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + ## tdnf config-manager --set-disable salt-repo-* + ## tdnf config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" - else - # Salt 3006 repo - echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" - echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" - echo "priority=10" >> "${YUM_REPO_FILE}" - echo "enabled=1" >> "${YUM_REPO_FILE}" - echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" - echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -6074,13 +6359,13 @@ __install_saltstack_photon_onedir_repository() { ## tdnf config-manager --set-enabled salt-repo-latest echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi tdnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then @@ -6109,7 +6394,7 @@ install_photon_deps() { __PACKAGES="${__PACKAGES} libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd sudo shadow" if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" @@ -6159,12 +6444,21 @@ install_photon_git_deps() { __PACKAGES="${__PACKAGES} git" fi + if ! __check_command_exists sudo; then + __PACKAGES="${__PACKAGES} sudo" + fi + + if ! __check_command_exists usermod; then + __PACKAGES="${__PACKAGES} shadow" + fi + if [ -n "${__PACKAGES}" ]; then # shellcheck disable=SC2086 __tdnf_install_noinput ${__PACKAGES} || return 1 __PACKAGES="" fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc glibc-devel linux-devel.x86_64 cython${PY_PKG_VER}" @@ -6206,6 +6500,8 @@ install_photon_git() { return 1 fi + install_photon_git_deps + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 else @@ -6314,7 +6610,7 @@ install_photon_onedir_deps() { __install_saltstack_photon_onedir_repository || return 1 fi - __PACKAGES="procps-ng" + __PACKAGES="procps-ng sudo shadow" # shellcheck disable=SC2086 __tdnf_install_noinput ${__PACKAGES} || return 1 @@ -6331,27 +6627,43 @@ install_photon_onedir_deps() { install_photon_onedir() { + echodebug "install_photon_onedir() entry" STABLE_REV=$ONEDIR_REV + _GENERIC_PKG_VERSION="" + + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + __get_packagesite_onedir_latest "$STABLE_REV" + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" + else + # default to latest version Salt, config and repo already setup + __get_packagesite_onedir_latest + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -6378,38 +6690,85 @@ install_photon_onedir_post() { # __ZYPPER_REQUIRES_REPLACE_FILES=-1 -__set_suse_pkg_repo() { - - # Set distro repo variable - if [ "${DISTRO_MAJOR_VERSION}" -gt 2015 ]; then - DISTRO_REPO="openSUSE_Tumbleweed" - elif [ "${DISTRO_MAJOR_VERSION}" -eq 15 ] && [ "${DISTRO_MINOR_VERSION}" -ge 4 ]; then - DISTRO_REPO="${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ] || [ "${DISTRO_MAJOR_VERSION}" -eq 15 ]; then - DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - else - DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" - fi - - suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" - suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" - SUSE_PKG_URL="$suse_pkg_url_base/$suse_pkg_url_path" -} __check_and_refresh_suse_pkg_repo() { # Check to see if systemsmanagement_saltstack exists - __zypper repos | grep -q systemsmanagement_saltstack + __zypper repos | grep -q 'salt.repo' if [ $? -eq 1 ]; then - # zypper does not yet know anything about systemsmanagement_saltstack - __zypper addrepo --refresh "${SUSE_PKG_URL}" || return 1 + # zypper does not yet know anything about salt.repo + # zypper does not support exclude similar to Photon, hence have to do following + ZYPPER_REPO_FILE="/etc/zypp/repos.d/salt.repo" + # shellcheck disable=SC2129 + if [ "$ONEDIR_REV" != "latest" ]; then + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + echo "[salt-repo-3007-sts]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}" + zypper addlock "salt-* < 3007" && zypper addlock "salt-* >= 3008" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}" + zypper addlock "salt-* < 3006" && zypper addlock "salt-* >= 3007" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}"a + ONEDIR_MAJ_VER=$(echo "${ONEDIR_REV_DOT}" | awk -F '.' '{print $1}') + # shellcheck disable=SC2004 + ONEDIR_MAJ_VER_PLUS=$((${ONEDIR_MAJ_VER} + 1)) + zypper addlock "salt-* < ${ONEDIR_MAJ_VER}" && zypper addlock "salt-* >= ${ONEDIR_MAJ_VER_PLUS}" + fi + else + # Enable the Salt LATEST repo + echo "[salt-repo-latest]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}" + fi + __zypper addrepo --refresh "${ZYPPER_REPO_FILE}" || return 1 fi } __version_lte() { - if ! __check_command_exists python; then - zypper --non-interactive install --replacefiles --auto-agree-with-licenses python || \ - zypper --non-interactive install --auto-agree-with-licenses python || return 1 + if ! __check_command_exists python3; then + zypper --non-interactive install --replacefiles --auto-agree-with-licenses python3 || \ + zypper --non-interactive install --auto-agree-with-licenses python3 || return 1 fi if [ "$(${_PY_EXE} -c 'import sys; V1=tuple([int(i) for i in sys.argv[1].split(".")]); V2=tuple([int(i) for i in sys.argv[2].split(".")]); print(V1<=V2)' "$1" "$2")" = "True" ]; then @@ -6453,8 +6812,6 @@ __zypper_install() { __opensuse_prep_install() { # DRY function for common installation preparatory steps for SUSE if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then - # Is the repository already known - __set_suse_pkg_repo # Check zypper repos and refresh if necessary __check_and_refresh_suse_pkg_repo fi @@ -6509,6 +6866,7 @@ install_opensuse_git_deps() { __zypper_install git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 # Check for Tumbleweed @@ -6535,23 +6893,34 @@ install_opensuse_onedir_deps() { } install_opensuse_stable() { + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -6733,6 +7102,7 @@ install_opensuse_15_git_deps() { __zypper_install git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -6844,12 +7214,94 @@ install_suse_15_restart_daemons() { return 0 } +install_suse_15_check_services() { + install_opensuse_check_services || return 1 + return 0 +} + # # End of SUSE Enterprise 15 # ####################################################################################################################### +####################################################################################################################### +# +# SUSE Enterprise 15, now has ID sled +# + +install_sled_15_stable_deps() { + __opensuse_prep_install || return 1 + install_opensuse_15_stable_deps || return 1 + + return 0 +} + +install_sled_15_git_deps() { + install_suse_15_stable_deps || return 1 + + if ! __check_command_exists git; then + __zypper_install git-core || return 1 + fi + + install_opensuse_15_git_deps || return 1 + + return 0 +} + +install_sled_15_onedir_deps() { + __opensuse_prep_install || return 1 + install_opensuse_15_onedir_deps || return 1 + + return 0 +} + +install_sled_15_stable() { + install_opensuse_stable || return 1 + return 0 +} + +install_sled_15_git() { + install_opensuse_15_git || return 1 + return 0 +} + +install_sled_15_onedir() { + install_opensuse_stable || return 1 + return 0 +} + +install_sled_15_stable_post() { + install_opensuse_stable_post || return 1 + return 0 +} + +install_sled_15_git_post() { + install_opensuse_git_post || return 1 + return 0 +} + +install_sled_15_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + +install_sled_15_restart_daemons() { + install_opensuse_restart_daemons || return 1 + return 0 +} + +install_sled_15_check_services() { + install_opensuse_check_services || return 1 + return 0 +} + +# +# End of SUSE Enterprise 15 aka sled +# +####################################################################################################################### + + ####################################################################################################################### # # Gentoo Install Functions. @@ -6988,6 +7440,7 @@ install_gentoo_git_deps() { echoinfo "Running emerge -v1 setuptools" __emerge -v1 setuptools || return 1 + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __gentoo_post_dep || return 1 } @@ -7258,7 +7711,7 @@ daemons_running_voidlinux() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __macosx_get_packagesite_onedir_latest -# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS +# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { @@ -7268,9 +7721,15 @@ __macosx_get_packagesite_onedir_latest() { macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 - wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" - # shellcheck disable=SC2010 - _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + wget -q -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" + if [ "$#" -gt 0 ] && [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi cd ${curr_pwd} || return "${_PKG_VERSION}" rm -fR ${macos_versions_tmpdir} @@ -7294,11 +7753,15 @@ __macosx_get_packagesite_onedir() { _ONEDIR_TYPE="saltproject-generic" SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - __macosx_get_packagesite_onedir_latest - elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then - _PKG_VERSION=$_ONEDIR_REV + __macosx_get_packagesite_onedir_latest + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # need to get latest for major version + __macosx_get_packagesite_onedir_latest "$_ONEDIR_REV" + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then + _PKG_VERSION=$_ONEDIR_REV else - __macosx_get_packagesite_onedir_latest + # default to getting latest + __macosx_get_packagesite_onedir_latest fi PKG="salt-${_PKG_VERSION}-py3-${DARWIN_ARCH}.pkg" @@ -7346,6 +7809,7 @@ install_macosx_git_deps() { # Install PIP $_PYEXE /tmp/get-pip.py || return 1 + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 return 0 @@ -7882,6 +8346,7 @@ fi if [ "${ITYPE}" = "git" ] && [ ${_NO_DEPS} -eq ${BS_TRUE} ]; then + # shellcheck disable=SC2119 if ! __git_clone_and_checkout; then echo "Failed to clone and checkout git repository." exit 1 From 2be143d902f456a89076073128c8bdb1d0a2286e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Mar 2025 13:22:28 -0400 Subject: [PATCH 06/31] upgrade salt 3006.10 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index e133dbd0b..5b989b295 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.9 + version: 3006.10 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index be405b9e8..e56bf02c0 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.9 + version: 3006.10 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From 028297cef8ecda720def39fa762e697fce2b20ee Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Mar 2025 13:46:30 -0400 Subject: [PATCH 07/31] add bootstrap-salt to preloaded soup_scripts --- salt/common/soup_scripts.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 60215f949..24e6c6152 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -64,6 +64,12 @@ copy_so-repo-sync_manager_tools_sbin: - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync - preserve: True +copy_bootstrap-salt_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh + - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh + - preserve: True + # This section is used to put the new script in place so that it can be called during soup. # It is faster than calling the states that normally manage them to put them in place. copy_so-common_sbin: @@ -108,6 +114,13 @@ copy_so-repo-sync_sbin: - force: True - preserve: True +copy_bootstrap-salt_sbin: + file.copy: + - name: /usr/sbin/bootstrap-salt.sh + - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh + - force: True + - preserve: True + {# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #} {% if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %} {% set saltrepofile = '/etc/yum.repos.d/salt.repo' %} From aa8fd647b655cb705e59fceb7fac5ec84df1ccb6 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Mar 2025 15:27:52 -0400 Subject: [PATCH 08/31] make string to not drop 0 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 5b989b295..97e29d0a8 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.10 + version: '3006.10' diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index e56bf02c0..d3cb54692 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.10 + version: '3006.10' check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From 41c0a91d77b92320b70a79c6b278d280171e6dc7 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Mar 2025 15:42:16 -0400 Subject: [PATCH 09/31] ensure versions are strings --- salt/salt/map.jinja | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 8feb06763..d77e23100 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -1,19 +1,23 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% import_yaml 'salt/minion.defaults.yaml' as saltminion %} -{% set SALTVERSION = saltminion.salt.minion.version %} +{% set SALTVERSION = saltminion.salt.minion.version | string %} +{% set INSTALLEDSALTVERSION = grains.saltversion | string %} {% if grains.os_family == 'Debian' %} {% set SPLITCHAR = '+' %} - {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} + {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion', 'salt-cloud'] %} {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} - {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} + {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion', 'salt-cloud'] %} {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} -{% set INSTALLEDSALTVERSION = grains.saltversion %} - -{% if grains.saltversion|string != SALTVERSION|string %} +{% if INSTALLEDSALTVERSION != SALTVERSION %} {% if grains.os_family|lower == 'redhat' %} {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -s 120 -r -F stable ' ~ SALTVERSION %} {% elif grains.os_family|lower == 'debian' %} From c30cbf9af02e4068d98290a77c6040e927a12bb7 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Mar 2025 15:44:56 -0400 Subject: [PATCH 10/31] remove salt-cloud --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index d77e23100..19571edce 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -9,11 +9,11 @@ {% if grains.os_family == 'Debian' %} {% set SPLITCHAR = '+' %} - {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion', 'salt-cloud'] %} + {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} - {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion', 'salt-cloud'] %} + {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} From 053eadbb39b0420515f632968b18b53183e7b86b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Mar 2025 16:58:16 -0400 Subject: [PATCH 11/31] fix SALTVERSION grep to work with or without quote --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fffa1a932..b97d4bb52 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -95,7 +95,7 @@ analyze_system() { desktop_salt_local() { - SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //') + SALTVERSION=$(grep "version:" ../salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+") # Install everything using local salt # Set the repo securityonion_repo @@ -1817,7 +1817,7 @@ securityonion_repo() { } repo_sync_local() { - SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //') + SALTVERSION=$(grep "version:" ../salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+") info "Repo Sync" if [[ $is_supported ]]; then # Sync the repo from the the SO repo locally. @@ -1878,7 +1878,7 @@ repo_sync_local() { saltify() { info "Installing Salt" - SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //') + SALTVERSION=$(grep "version:" ../salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+") if [[ $is_deb ]]; then DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || fail_setup From 760ff1e45b051846e79049e9ad2d4df55737f2fe Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 21 Mar 2025 08:20:04 -0400 Subject: [PATCH 12/31] work with quotes in version --- salt/manager/tools/sbin/soup | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3d5ca338f..d49c53fed 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1085,7 +1085,7 @@ upgrade_check() { } upgrade_check_salt() { - NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}') + NEWSALTVERSION=$(grep "version:" $UPDATE_DIR/salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+") if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then echo "You are already running the correct version of Salt for Security Onion." else @@ -1231,26 +1231,6 @@ failed_soup_restore_items() { masterunlock } -#upgrade salt to 3004.1 -#2_3_10_hotfix_1() { -# systemctl_func "stop" "$cron_service_name" -# # update mine items prior to stopping salt-minion and salt-master -# update_salt_mine -# stop_salt_minion -# stop_salt_master -# update_repo -# # Does salt need upgraded. If so update it. -# if [[ $UPGRADESALT -eq 1 ]]; then -# echo "Upgrading Salt" -# # Update the repo files so it can actually upgrade -# upgrade_salt -# fi -# systemctl_func "start" "salt-master" -# systemctl_func "start" "salt-minion" -# systemctl_func "start" "$cron_service_name" - -#} - main() { trap 'check_err $?' EXIT From 1236c8c1f2630e7a82e31a503b9a64234c13e212 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 21 Mar 2025 10:34:55 -0400 Subject: [PATCH 13/31] support pcap imports for sensors in distributed grids --- salt/soc/files/bin/salt-relay.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index c9e3ba8c8..16c387f86 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -283,7 +283,7 @@ function send_file() { log "encrypting..." password=$(lookup_pillar_secret import_pass) - response=$(gpg --passphrase "$password" --batch --symmetric --cipher-algo AES256 "$from") + response=$(gpg --passphrase "$password" --batch --yes --symmetric --cipher-algo AES256 "$from") log Response:$'\n'"$response" fromgpg="$from.gpg" @@ -329,12 +329,11 @@ function import_file() { log "decrypting..." password=$(lookup_pillar_secret import_pass) - decrypt_cmd="gpg --passphrase $password -o $file.tmp --batch --decrypt $filegpg" + decrypt_cmd="gpg --passphrase $password -o $file --batch --yes --decrypt $filegpg" salt "$node" cmd.run "\"$decrypt_cmd\"" decrypt_code=$? if [[ $decrypt_code -eq 0 ]]; then - mv "$file.tmp" "$file" log "importing..." case $importer in pcap) @@ -357,7 +356,7 @@ function import_file() { exit_code=$decrypt_code fi - rm -f "$file" "$filegpg" + salt "$node" cmd.run "rm -f \"$file\" \"$filegpg\"" log Response:$'\n'"$response" log "Exit Code: $exit_code" From ed233401574f18e6634d99072a1db03715380450 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 21 Mar 2025 14:48:31 -0400 Subject: [PATCH 14/31] move pcapoutdir --- salt/pcap/init.sls | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 7a172e8fd..20b9bc2b7 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -23,4 +23,11 @@ pcapdir: - name: /nsm/pcap - user: 941 - group: 941 - - makedirs: True \ No newline at end of file + - makedirs: True + +pcapoutdir: + file.directory: + - name: /nsm/pcapout + - user: 939 + - group: 939 + - makedirs: True From a0637fa25dea6de9260c23c0f0948cc7abd94bb6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 21 Mar 2025 14:54:52 -0400 Subject: [PATCH 15/31] ignore false positives --- salt/common/tools/sbin/so-log-check | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 44e8360e2..66f6825e9 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -127,6 +127,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished" # Telegraf script finished just as the auto kill timeout kicked in EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available" # Typical error when making a query before ES has finished loading all indices EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -155,6 +156,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request_unauthorized" # false positive (login failures to Hydra result in an 'error' log) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index lifecycle policy" # false positive (elasticsearch policy names contain 'error') EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline" # false positive (elasticsearch ingest pipeline names contain 'error') + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template" # false positive (elasticsearch index or template names contain 'error') fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then From 844283cc38d69bfef18f1f6cfc98ba409f7bcdce Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 21 Mar 2025 14:55:52 -0500 Subject: [PATCH 16/31] get more results --- salt/elasticfleet/tools/sbin/so-elastic-fleet-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 7e1e4b790..d8d0bdb1e 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -108,7 +108,7 @@ elastic_fleet_package_is_installed() { } elastic_fleet_installed_packages() { - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=300" + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500" } elastic_fleet_agent_policy_ids() { From d0bb86a24fa5f97330f1a830d6e2fee8738927fd Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 22 Mar 2025 07:12:19 -0400 Subject: [PATCH 17/31] Remove pcapoutdir --- salt/pcap/config.sls | 7 ------- 1 file changed, 7 deletions(-) diff --git a/salt/pcap/config.sls b/salt/pcap/config.sls index eb37765c5..173fecfd1 100644 --- a/salt/pcap/config.sls +++ b/salt/pcap/config.sls @@ -79,13 +79,6 @@ pcaptmpdir: - group: 941 - makedirs: True -pcapoutdir: - file.directory: - - name: /nsm/pcapout - - user: 939 - - group: 939 - - makedirs: True - pcapindexdir: file.directory: - name: /nsm/pcapindex From 0952b7528ff6f6829640aad2a0ab3bcf01afc33b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Sat, 22 Mar 2025 12:57:13 -0400 Subject: [PATCH 18/31] update mine update mine after salt-master restart and before highstate --- salt/manager/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8616f8489..a0dd5916f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1439,6 +1439,9 @@ main() { # Stop long-running scripts to allow potentially updated scripts to load on the next execution. killall salt-relay.sh + # ensure the mine is updated and populated before highstates run, following the salt-master restart + update_salt_mine + highstate postupgrade_changes [[ $is_airgap -eq 0 ]] && unmount_update From b974c6e8dffc5ebc4ca4b2d3b35c068846eae627 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Sun, 23 Mar 2025 12:07:39 -0400 Subject: [PATCH 19/31] roll back to 3006.9 but leave prep in place for future upgrades --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 97e29d0a8..8e1a618fd 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: '3006.10' + version: '3006.9' diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index d3cb54692..68ae04804 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: '3006.10' + version: '3006.9' check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From e61d37893aa612d625ac1f0f5e562b0bd8c62a3c Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 24 Mar 2025 12:33:10 -0400 Subject: [PATCH 20/31] start salt-minion service when mainint has ip --- salt/salt/minion.defaults.yaml | 1 - salt/salt/minion.sls | 3 --- salt/salt/service/salt-minion.service.jinja | 3 ++- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 68ae04804..7ec839950 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -3,4 +3,3 @@ salt: minion: version: '3006.9' check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default - service_start_delay: 30 # in seconds. diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index ee7c1f14c..ae2f9f487 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -5,7 +5,6 @@ {% from 'salt/map.jinja' import SALTPACKAGES %} {% from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %} {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} -{% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} include: - salt.python_modules @@ -89,8 +88,6 @@ salt_minion_service_unit_file: - name: {{ SYSTEMD_UNIT_FILE }} - source: salt://salt/service/salt-minion.service.jinja - template: jinja - - defaults: - service_start_delay: {{ service_start_delay }} - onchanges_in: - module: systemd_reload diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja index 27c7a15b6..2763b30f5 100644 --- a/salt/salt/service/salt-minion.service.jinja +++ b/salt/salt/service/salt-minion.service.jinja @@ -8,8 +8,9 @@ KillMode=process Type=notify NotifyAccess=all LimitNOFILE=8192 +ExecStartPre=/bin/bash -c 'until /sbin/ip -4 addr show dev {{ salt["pillar.get"]("host:mainint") }} | grep -q "inet "; do sleep 1; done' ExecStart=/usr/bin/salt-minion -ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} +TimeoutStartSec=120 [Install] WantedBy=multi-user.target From dcb667b32d01b19230383aeb61526d42948f7c83 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 24 Mar 2025 13:35:39 -0400 Subject: [PATCH 21/31] 2.4.140 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++---------- sigs/securityonion-2.4.140-20250324.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.140-20250324.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index c84462fd6..812d4c2a7 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.130-20250311 ISO image released on 2025/03/11 +### 2.4.140-20250324 ISO image released on 2025/03/24 ### Download and Verify -2.4.130-20250311 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.130-20250311.iso +2.4.140-20250324 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.140-20250324.iso -MD5: 4641CA710570CCE18CD7D50653373DC0 -SHA1: 786EF73F7945FDD80126C9AE00BDD29E58743715 -SHA256: 48C7A042F20C46B8087BAE0F971696DADE9F9364D52F416718245C16E7CCB977 +MD5: 36393200A5CEEC5B58277691DDAFF247 +SHA1: 48655378C732CF47A6B3290F6F07F4F3162BE054 +SHA256: 470E00245EBAD83C045743CFB27885CEC3E1F057D91081906B240A38B6D3759A Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.130-20250311.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.140-20250324.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.130-20250311.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.140-20250324.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.130-20250311.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.140-20250324.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.130-20250311.iso.sig securityonion-2.4.130-20250311.iso +gpg --verify securityonion-2.4.140-20250324.iso.sig securityonion-2.4.140-20250324.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 10 Mar 2025 06:30:49 PM EDT using RSA key ID FE507013 +gpg: Signature made Sun 23 Mar 2025 08:37:47 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.140-20250324.iso.sig b/sigs/securityonion-2.4.140-20250324.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..ca910c2c246a489aa60efef0434802e60e08ac16 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%zcu_xb%urJr zhyU>CY^mU7a4`Hd9DoJUDHOiIF|^!(P53$)6nR49_+20e~Cdi;-eT=SjQF^uHWPjQ~?4MfbnM| zh3W|eyEQ=X!(<4-Q{)l226GBS*&)K3n2~Bjsd^G2o`iTn9&q+fq4S_!PBfbj6Z&p0 zt{%lHo}+dV4}KaadkU%EM|a#$=>)z*b|VHxN~_65VOwZ|sXPR_xJtOhsItqgSS?_r E+Hi;ofB*mh literal 0 HcmV?d00001 From 1f98cef8162bcd221106fd6bf63f24f050c5bc65 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 24 Mar 2025 15:15:57 -0400 Subject: [PATCH 22/31] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bbcd1a024..316ab4cee 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.140 +2.4.150 From 54f3a8cb9110be165fc35f63f0a94d8ed99585a6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 24 Mar 2025 15:16:43 -0400 Subject: [PATCH 23/31] Update 2-4.yml --- .github/DISCUSSION_TEMPLATE/2-4.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/DISCUSSION_TEMPLATE/2-4.yml b/.github/DISCUSSION_TEMPLATE/2-4.yml index 3ba6cbd94..e6aec285d 100644 --- a/.github/DISCUSSION_TEMPLATE/2-4.yml +++ b/.github/DISCUSSION_TEMPLATE/2-4.yml @@ -26,6 +26,7 @@ body: - 2.4.120 - 2.4.130 - 2.4.140 + - 2.4.150 - Other (please provide detail below) validations: required: true From 0fbb6afee19fe7009eb31458519456da1b132d48 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 24 Mar 2025 15:51:22 -0400 Subject: [PATCH 24/31] soup for 2.4.150 --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a0dd5916f..19cf4af40 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -408,6 +408,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120 [[ "$INSTALLEDVERSION" == 2.4.120 ]] && up_to_2.4.130 [[ "$INSTALLEDVERSION" == 2.4.130 ]] && up_to_2.4.140 + [[ "$INSTALLEDVERSION" == 2.4.140 ]] && up_to_2.4.150 true } @@ -433,6 +434,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.111 ]] && post_to_2.4.120 [[ "$POSTVERSION" == 2.4.120 ]] && post_to_2.4.130 [[ "$POSTVERSION" == 2.4.130 ]] && post_to_2.4.140 + [[ "$POSTVERSION" == 2.4.140 ]] && post_to_2.4.150 true } @@ -560,6 +562,11 @@ post_to_2.4.140() { POSTVERSION=2.4.140 } +post_to_2.4.150() { + echo "Nothing to apply" + POSTVERSION=2.4.150 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -786,6 +793,12 @@ up_to_2.4.140() { INSTALLEDVERSION=2.4.140 } +up_to_2.4.150() { + echo "Nothing to do for 2.4.150" + + INSTALLEDVERSION=2.4.150 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls From 8f40b66e3b7f04759a6b4c5dc0ef28c0beb6a207 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 24 Mar 2025 19:49:24 -0400 Subject: [PATCH 25/31] update mine instead of failing highstate if no node_data --- pillar/node_data/ips.sls | 4 ++++ salt/mine/update.sls | 7 +++++++ salt/orch/mine_update.sls | 21 +++++++++++++++++++++ salt/reactor/mine_update.sls | 8 ++++++++ salt/salt/master.sls | 1 + salt/top.sls | 30 +++++++++++++++++------------- 6 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 salt/mine/update.sls create mode 100644 salt/orch/mine_update.sls create mode 100644 salt/reactor/mine_update.sls diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls index 5801d36f1..a2528a476 100644 --- a/pillar/node_data/ips.sls +++ b/pillar/node_data/ips.sls @@ -24,6 +24,7 @@ {% endif %} {% endfor %} +{% if node_types %} node_data: {% for node_type, host_values in node_types.items() %} {% for hostname, details in host_values.items() %} @@ -33,3 +34,6 @@ node_data: role: {{node_type}} {% endfor %} {% endfor %} +{% else %} +node_data: False +{% endif %} diff --git a/salt/mine/update.sls b/salt/mine/update.sls new file mode 100644 index 000000000..59e137d2b --- /dev/null +++ b/salt/mine/update.sls @@ -0,0 +1,7 @@ +# This state sends an event to the salt-master event bus +# The event will be caught by the reactor and trigger the mine_update orchestration + +send_mine_update_event: + module.run: + - name: event.send + - tag: salt/minion/{{grains.id}}/mine_update diff --git a/salt/orch/mine_update.sls b/salt/orch/mine_update.sls new file mode 100644 index 000000000..1483c1225 --- /dev/null +++ b/salt/orch/mine_update.sls @@ -0,0 +1,21 @@ +# Get the minion ID from the pillar +{% set MINION_ID = salt['pillar.get']('minion_id') %} + +# Run mine.update on all minions +update_mine_all_minions: + salt.function: + - name: mine.update + - tgt: '*' + - batch: 50 + - retry: + attempts: 3 + interval: 1 + +# Run highstate on the original minion +run_highstate_on_original_minion: + salt.state: + - tgt: {{ MINION_ID }} + - highstate: True + - queue: True + - require: + - salt: update_mine_all_minions diff --git a/salt/reactor/mine_update.sls b/salt/reactor/mine_update.sls new file mode 100644 index 000000000..968987cec --- /dev/null +++ b/salt/reactor/mine_update.sls @@ -0,0 +1,8 @@ +# This reactor triggers the mine_update orchestration when it receives a mine_update event + +trigger_mine_update_orchestration: + runner.state.orchestrate: + - args: + - mods: orch.mine_update + - pillar: + minion_id: {{ data['id'] }} diff --git a/salt/salt/master.sls b/salt/salt/master.sls index cf9f4718c..ddfb9c3e7 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -3,6 +3,7 @@ include: - salt.minion + - salt.master.reactor_config_mine_update hold_salt_master_package: module.run: diff --git a/salt/top.sls b/salt/top.sls index 437c44bf8..552cd1ea7 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -22,7 +22,11 @@ base: - salt.minion-state-apply-test - salt.minion - '* and G@saltversion:{{saltversion}}': + '*_eval or *_manager* or *_standalone or *_import and I@node_data:False': + - match: compound + - mine.update + + '* and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.minion - patch.os.schedule @@ -33,7 +37,7 @@ base: - docker - docker_clean - '*_sensor and G@saltversion:{{saltversion}}': + '*_sensor and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - sensor - ssl @@ -49,7 +53,7 @@ base: - elasticfleet.install_agent_grid - stig - '*_eval and G@saltversion:{{saltversion}}': + '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - sensor @@ -81,7 +85,7 @@ base: - utility - elasticfleet - '*_manager and G@saltversion:{{saltversion}}': + '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - ca @@ -112,7 +116,7 @@ base: - stig - kafka - '*_standalone and G@saltversion:{{saltversion}}': + '*_standalone and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - sensor @@ -148,7 +152,7 @@ base: - stig - kafka - '*_searchnode and G@saltversion:{{saltversion}}': + '*_searchnode and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - firewall - ssl @@ -161,7 +165,7 @@ base: - stig - kafka - '*_managersearch and G@saltversion:{{saltversion}}': + '*_managersearch and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - ca @@ -192,7 +196,7 @@ base: - stig - kafka - '*_heavynode and G@saltversion:{{saltversion}}': + '*_heavynode and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - sensor - ssl @@ -211,7 +215,7 @@ base: - elasticfleet.install_agent_grid - elasticagent - '*_import and G@saltversion:{{saltversion}}': + '*_import and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - sensor @@ -239,7 +243,7 @@ base: - zeek - elasticfleet - '*_receiver and G@saltversion:{{saltversion}}': + '*_receiver and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - ssl - sensoroni @@ -251,7 +255,7 @@ base: - kafka - stig - '*_idh and G@saltversion:{{saltversion}}': + '*_idh and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - ssl - sensoroni @@ -260,7 +264,7 @@ base: - elasticfleet.install_agent_grid - idh - '*_fleet and G@saltversion:{{saltversion}}': + '*_fleet and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - ssl - sensoroni @@ -272,7 +276,7 @@ base: - elasticfleet.install_agent_grid - schedule - '*_desktop and G@saltversion:{{saltversion}}': + '*_desktop and G@saltversion:{{saltversion}} and not I@node_data:False': - ssl - sensoroni - telegraf From d7e831fbeb55ca42aea7c96651ac6d456443d7ba Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 24 Mar 2025 20:45:35 -0400 Subject: [PATCH 26/31] add mine_update reactor config for master --- salt/salt/master/reactor_config_mine_update.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 salt/salt/master/reactor_config_mine_update.sls diff --git a/salt/salt/master/reactor_config_mine_update.sls b/salt/salt/master/reactor_config_mine_update.sls new file mode 100644 index 000000000..deffbf58d --- /dev/null +++ b/salt/salt/master/reactor_config_mine_update.sls @@ -0,0 +1,13 @@ +reactor_config_hypervisor: + file.managed: + - name: /etc/salt/master.d/reactor_mine_update.conf + - contents: | + reactor: + - 'salt/minion/*/mine_update': + - salt://reactor/mine_update.sls + - group: root + - mode: 644 + - makedirs: True + - watch_in: + - service: salt_master_service + - order: last From 79388af6450a38806f5d14f4ee2a11a0857756b8 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 25 Mar 2025 10:17:43 -0400 Subject: [PATCH 27/31] only managers need node_ips --- pillar/top.sls | 6 +- salt/top.sls | 158 ++++++++++++++++++++++++------------------------- 2 files changed, 84 insertions(+), 80 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index b8d694e23..33b5feb2d 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -24,10 +24,10 @@ base: - firewall.adv_firewall - nginx.soc_nginx - nginx.adv_nginx - - node_data.ips '*_manager or *_managersearch': - match: compound + - node_data.ips {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -90,6 +90,7 @@ base: - soc.license '*_eval': + - node_data.ips - secrets - healthcheck.eval - elasticsearch.index_templates @@ -138,6 +139,7 @@ base: - minions.adv_{{ grains.id }} '*_standalone': + - node_data.ips - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash @@ -260,6 +262,7 @@ base: - soc.license '*_import': + - node_data.ips - secrets - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} @@ -305,6 +308,7 @@ base: - minions.adv_{{ grains.id }} '*_fleet': + - node_data.ips - backup.soc_backup - backup.adv_backup - logstash.nodes diff --git a/salt/top.sls b/salt/top.sls index 552cd1ea7..d33b23932 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -22,11 +22,7 @@ base: - salt.minion-state-apply-test - salt.minion - '*_eval or *_manager* or *_standalone or *_import and I@node_data:False': - - match: compound - - mine.update - - '* and G@saltversion:{{saltversion}} and not I@node_data:False': + '* and G@saltversion:{{saltversion}}': - match: compound - salt.minion - patch.os.schedule @@ -37,21 +33,9 @@ base: - docker - docker_clean - '*_sensor and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_manager* or *_standalone or *_eval or *_import and I@node_data:False': - match: compound - - sensor - - ssl - - sensoroni - - telegraf - - firewall - - nginx - - pcap - - suricata - - healthcheck - - zeek - - strelka - - elasticfleet.install_agent_grid - - stig + - mine.update '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound @@ -85,37 +69,6 @@ base: - utility - elasticfleet - '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - - match: compound - - salt.master - - ca - - ssl - - registry - - nginx - - influxdb - - strelka.manager - - soc - - kratos - - hydra - - firewall - - manager - - sensoroni - - telegraf - - backup.config_backup - - idstools - - suricata.manager - - elasticsearch - - logstash - - redis - - elastic-fleet-package-registry - - kibana - - curator.disabled - - elastalert - - utility - - elasticfleet - - stig - - kafka - '*_standalone and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master @@ -152,16 +105,34 @@ base: - stig - kafka - '*_searchnode and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - - firewall + - salt.master + - ca - ssl - - elasticsearch - - logstash + - registry + - nginx + - influxdb + - strelka.manager + - soc + - kratos + - hydra + - firewall + - manager - sensoroni - telegraf - - nginx - - elasticfleet.install_agent_grid + - backup.config_backup + - idstools + - suricata.manager + - elasticsearch + - logstash + - redis + - elastic-fleet-package-registry + - kibana + - curator.disabled + - elastalert + - utility + - elasticfleet - stig - kafka @@ -196,25 +167,6 @@ base: - stig - kafka - '*_heavynode and G@saltversion:{{saltversion}} and not I@node_data:False': - - match: compound - - sensor - - ssl - - sensoroni - - telegraf - - nginx - - firewall - - elasticsearch - - logstash - - redis - - curator.disabled - - strelka - - pcap - - suricata - - zeek - - elasticfleet.install_agent_grid - - elasticagent - '*_import and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master @@ -243,7 +195,55 @@ base: - zeek - elasticfleet - '*_receiver and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_searchnode and G@saltversion:{{saltversion}}': + - match: compound + - firewall + - ssl + - elasticsearch + - logstash + - sensoroni + - telegraf + - nginx + - elasticfleet.install_agent_grid + - stig + - kafka + + '*_sensor and G@saltversion:{{saltversion}}': + - match: compound + - sensor + - ssl + - sensoroni + - telegraf + - firewall + - nginx + - pcap + - suricata + - healthcheck + - zeek + - strelka + - elasticfleet.install_agent_grid + - stig + + '*_heavynode and G@saltversion:{{saltversion}}': + - match: compound + - sensor + - ssl + - sensoroni + - telegraf + - nginx + - firewall + - elasticsearch + - logstash + - redis + - curator.disabled + - strelka + - pcap + - suricata + - zeek + - elasticfleet.install_agent_grid + - elasticagent + + '*_receiver and G@saltversion:{{saltversion}}': - match: compound - ssl - sensoroni @@ -255,7 +255,7 @@ base: - kafka - stig - '*_idh and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_idh and G@saltversion:{{saltversion}}': - match: compound - ssl - sensoroni @@ -264,7 +264,7 @@ base: - elasticfleet.install_agent_grid - idh - '*_fleet and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_fleet and G@saltversion:{{saltversion}}': - match: compound - ssl - sensoroni @@ -276,7 +276,7 @@ base: - elasticfleet.install_agent_grid - schedule - '*_desktop and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_desktop and G@saltversion:{{saltversion}}': - ssl - sensoroni - telegraf From 55c815cae8c043775442e7b549d361730ce33167 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 25 Mar 2025 19:44:38 -0400 Subject: [PATCH 28/31] simplify highstate rerun when node_data pillar empty --- salt/mine/update.sls | 27 ++++++++++++++++--- salt/orch/mine_update.sls | 21 --------------- salt/reactor/mine_update.sls | 8 ------ salt/salt/master.sls | 1 - .../master/reactor_config_mine_update.sls | 13 --------- salt/top.sls | 13 ++++----- 6 files changed, 30 insertions(+), 53 deletions(-) delete mode 100644 salt/orch/mine_update.sls delete mode 100644 salt/reactor/mine_update.sls delete mode 100644 salt/salt/master/reactor_config_mine_update.sls diff --git a/salt/mine/update.sls b/salt/mine/update.sls index 59e137d2b..d7cdd18d0 100644 --- a/salt/mine/update.sls +++ b/salt/mine/update.sls @@ -1,7 +1,26 @@ # This state sends an event to the salt-master event bus # The event will be caught by the reactor and trigger the mine_update orchestration -send_mine_update_event: - module.run: - - name: event.send - - tag: salt/minion/{{grains.id}}/mine_update +{# may be able to use this method if we can figure out multi state run failure - https://github.com/saltstack/salt/issues/66929 #} +# Get the minion ID from the pillar +{% set MINION_ID = grains.id %} + +# Run mine.update on all minions +mine.update.update_mine_all_minions: + salt.function: + - name: mine.update + - tgt: '*' + - batch: 50 + - retry: + attempts: 3 + interval: 1 + +# Run highstate on the original minion +# we can use concurrent on this highstate because no other highstate would be running when this is called +mine.update.run_highstate_on_{{ MINION_ID }}: + salt.state: + - tgt: {{ MINION_ID }} + - highstate: True + - concurrent: True + - require: + - salt: mine.update.update_mine_all_minions diff --git a/salt/orch/mine_update.sls b/salt/orch/mine_update.sls deleted file mode 100644 index 1483c1225..000000000 --- a/salt/orch/mine_update.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Get the minion ID from the pillar -{% set MINION_ID = salt['pillar.get']('minion_id') %} - -# Run mine.update on all minions -update_mine_all_minions: - salt.function: - - name: mine.update - - tgt: '*' - - batch: 50 - - retry: - attempts: 3 - interval: 1 - -# Run highstate on the original minion -run_highstate_on_original_minion: - salt.state: - - tgt: {{ MINION_ID }} - - highstate: True - - queue: True - - require: - - salt: update_mine_all_minions diff --git a/salt/reactor/mine_update.sls b/salt/reactor/mine_update.sls deleted file mode 100644 index 968987cec..000000000 --- a/salt/reactor/mine_update.sls +++ /dev/null @@ -1,8 +0,0 @@ -# This reactor triggers the mine_update orchestration when it receives a mine_update event - -trigger_mine_update_orchestration: - runner.state.orchestrate: - - args: - - mods: orch.mine_update - - pillar: - minion_id: {{ data['id'] }} diff --git a/salt/salt/master.sls b/salt/salt/master.sls index ddfb9c3e7..cf9f4718c 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -3,7 +3,6 @@ include: - salt.minion - - salt.master.reactor_config_mine_update hold_salt_master_package: module.run: diff --git a/salt/salt/master/reactor_config_mine_update.sls b/salt/salt/master/reactor_config_mine_update.sls deleted file mode 100644 index deffbf58d..000000000 --- a/salt/salt/master/reactor_config_mine_update.sls +++ /dev/null @@ -1,13 +0,0 @@ -reactor_config_hypervisor: - file.managed: - - name: /etc/salt/master.d/reactor_mine_update.conf - - contents: | - reactor: - - 'salt/minion/*/mine_update': - - salt://reactor/mine_update.sls - - group: root - - mode: 644 - - makedirs: True - - watch_in: - - service: salt_master_service - - order: last diff --git a/salt/top.sls b/salt/top.sls index d33b23932..4a6e8e010 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -17,12 +17,17 @@ base: - schedule - logrotate - 'not G@saltversion:{{saltversion}}': + 'I@node_data:False and ( *_manager* or *_eval or *_import or *_standalone )': + - match: compound + - salt.minion + - mine.update + + 'not G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.minion-state-apply-test - salt.minion - '* and G@saltversion:{{saltversion}}': + '* and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.minion - patch.os.schedule @@ -33,10 +38,6 @@ base: - docker - docker_clean - '*_manager* or *_standalone or *_eval or *_import and I@node_data:False': - - match: compound - - mine.update - '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master From 5836bc5bd1daefd71fe2f890bebc13a4b005e857 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 25 Mar 2025 21:58:42 -0400 Subject: [PATCH 29/31] remove require since maybe some failure from mine.update --- salt/mine/update.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/mine/update.sls b/salt/mine/update.sls index d7cdd18d0..03ea76c9e 100644 --- a/salt/mine/update.sls +++ b/salt/mine/update.sls @@ -22,5 +22,3 @@ mine.update.run_highstate_on_{{ MINION_ID }}: - tgt: {{ MINION_ID }} - highstate: True - concurrent: True - - require: - - salt: mine.update.update_mine_all_minions From bb8f0605e1e7451d726cce80de746c95d7161206 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 26 Mar 2025 10:50:04 -0400 Subject: [PATCH 30/31] patch x509_v2 state salt issue 66929 --- salt/salt/minion.sls | 1 + salt/salt/patch/x509_v2/init.sls | 6 + setup/files/patch/states/x509_v2.py | 1814 --------------------------- setup/so-functions | 7 +- 4 files changed, 9 insertions(+), 1819 deletions(-) create mode 100644 salt/salt/patch/x509_v2/init.sls delete mode 100644 setup/files/patch/states/x509_v2.py diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index ae2f9f487..9f2a6f3fa 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -8,6 +8,7 @@ include: - salt.python_modules + - salt.patch.x509_v2 - salt - systemd.reload - repo.client diff --git a/salt/salt/patch/x509_v2/init.sls b/salt/salt/patch/x509_v2/init.sls new file mode 100644 index 000000000..2483339a2 --- /dev/null +++ b/salt/salt/patch/x509_v2/init.sls @@ -0,0 +1,6 @@ +patch_x509_v2_state_module: + file.replace: + - name: /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py + - pattern: 'res = __salt__\["state.single"\]\("file.managed", name, test=test, \*\*kwargs\)' + - repl: 'res = __salt__["state.single"]("file.managed", name, test=test, concurrent=True, **kwargs)' + - backup: .bak diff --git a/setup/files/patch/states/x509_v2.py b/setup/files/patch/states/x509_v2.py deleted file mode 100644 index acc86045f..000000000 --- a/setup/files/patch/states/x509_v2.py +++ /dev/null @@ -1,1814 +0,0 @@ -""" -Manage X.509 certificates -========================= - -.. versionadded:: 3006.0 - - This module represents a complete rewrite of the original ``x509`` modules - and is named ``x509_v2`` since it introduces breaking changes. - -:depends: cryptography - -.. note:: - - All parameters that take a public key, private key, certificate, - CSR or CRL can be specified either as a PEM/hex/base64 string or - a path to a local file encoded in all supported formats for the type. - -Configuration instructions and general remarks are documented -in the :ref:`execution module docs `. - -For the list of breaking changes versus the previous ``x509`` modules, -please also refer to the :ref:`execution module docs `. - -About ------ -This module can enable managing a complete PKI infrastructure, including creating -private keys, CAs, certificates and CRLs. It includes the ability to generate a -private key on a server, and have the corresponding public key sent to a remote -CA to create a CA signed certificate. This can be done in a secure manner, where -private keys are always generated locally and never moved across the network. - -Example -------- -Here is a simple example scenario. In this example ``ca`` is the ca server, -and ``www`` is a web server that needs a certificate signed by ``ca``. - -.. note:: - - Remote signing requires the setup of :term:`Peer Communication` and signing - policies. Please see the :ref:`execution module docs `. - - -/srv/salt/top.sls - -.. code-block:: yaml - - base: - '*': - - cert - 'ca': - - ca - 'www': - - www - -This state creates the CA key, certificate and signing policy. It also publishes -the certificate to the mine, where it can be easily retrieved by other minions. - -.. code-block:: yaml - - # /srv/salt/ca.sls - - Configure the x509 module: - file.managed: - - name: /etc/salt/minion.d/x509.conf - - source: salt://x509.conf - - Restart Salt minion: - cmd.run: - - name: 'salt-call service.restart salt-minion' - - bg: true - - onchanges: - - file: /etc/salt/minion.d/x509.conf - - Ensure PKI directories exist: - file.directory: - - name: /etc/pki/issued_certs - - makedirs: true - - Create CA private key: - x509.private_key_managed: - - name: /etc/pki/ca.key - - keysize: 4096 - - backup: true - - require: - - file: /etc/pki - - Create self-signed CA certificate: - x509.certificate_managed: - - name: /etc/pki/ca.crt - - signing_private_key: /etc/pki/ca.key - - CN: ca.example.com - - C: US - - ST: Utah - - L: Salt Lake City - - basicConstraints: "critical, CA:true" - - keyUsage: "critical, cRLSign, keyCertSign" - - subjectKeyIdentifier: hash - - authorityKeyIdentifier: keyid:always,issuer - - days_valid: 3650 - - days_remaining: 0 - - backup: true - - require: - - x509: /etc/pki/ca.key - -.. code-block:: yaml - - # /srv/salt/x509.conf - - # enable x509_v2 - features: - x509_v2: true - - # publish the CA certificate to the mine - mine_functions: - x509.get_pem_entries: [/etc/pki/ca.crt] - - # define at least one signing policy for remote signing - x509_signing_policies: - www: - - minions: 'www' - - signing_private_key: /etc/pki/ca.key - - signing_cert: /etc/pki/ca.crt - - C: US - - ST: Utah - - L: Salt Lake City - - basicConstraints: "critical CA:false" - - keyUsage: "critical keyEncipherment" - - subjectKeyIdentifier: hash - - authorityKeyIdentifier: keyid:always,issuer - - days_valid: 30 - - copypath: /etc/pki/issued_certs/ - - -This example state will instruct all minions to trust certificates signed by -our new CA. Mind that this example works for Debian-based OS only. -Also note the Jinja call to encode the string to JSON, which will avoid -YAML issues with newline characters. - -.. code-block:: jinja - - # /srv/salt/cert.sls - - Ensure the CA trust bundle exists: - file.directory: - - name: /usr/local/share/ca-certificates - - Ensure our self-signed CA certificate is included: - x509.pem_managed: - - name: /usr/local/share/ca-certificates/myca.crt - - text: {{ salt["mine.get"]("ca", "x509.get_pem_entries")["ca"]["/etc/pki/ca.crt"] | json }} - -This state creates a private key, then requests a certificate signed by our CA -according to the www policy. - -.. code-block:: yaml - - # /srv/salt/www.sls - - Ensure PKI directory exists: - file.directory: - - name: /etc/pki - - Create private key for the certificate: - x509.private_key_managed: - - name: /etc/pki/www.key - - keysize: 4096 - - backup: true - - require: - - file: /etc/pki - - Request certificate: - x509.certificate_managed: - - name: /etc/pki/www.crt - - ca_server: ca - - signing_policy: www - - private_key: /etc/pki/www.key - - CN: www.example.com - - days_remaining: 7 - - backup: true - - require: - - x509: /etc/pki/www.key -""" -import base64 -import copy -import datetime -import logging -import os.path - -import salt.utils.context -import salt.utils.files -from salt.exceptions import CommandExecutionError, SaltInvocationError -from salt.features import features -from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS - -try: - import cryptography.x509 as cx509 - from cryptography.exceptions import UnsupportedAlgorithm - from cryptography.hazmat.primitives import hashes - - import salt.utils.x509 as x509util - - HAS_CRYPTOGRAPHY = True -except ImportError: - HAS_CRYPTOGRAPHY = False - - -log = logging.getLogger(__name__) - -__virtualname__ = "x509" - - -def __virtual__(): - if not HAS_CRYPTOGRAPHY: - return (False, "Could not load cryptography") - if not features.get("x509_v2"): - return ( - False, - "x509_v2 needs to be explicitly enabled by setting `x509_v2: true` " - "in the minion configuration value `features` until Salt 3008 (Argon).", - ) - return __virtualname__ - - -def certificate_managed( - name, - days_remaining=None, - ca_server=None, - signing_policy=None, - encoding="pem", - append_certs=None, - copypath=None, - prepend_cn=False, - digest="sha256", - signing_private_key=None, - signing_private_key_passphrase=None, - signing_cert=None, - public_key=None, - private_key=None, - private_key_passphrase=None, - csr=None, - subject=None, - serial_number=None, - not_before=None, - not_after=None, - days_valid=None, - pkcs12_passphrase=None, - pkcs12_encryption_compat=False, - pkcs12_friendlyname=None, - **kwargs, -): - """ - Ensure an X.509 certificate is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_certificate `, - as well as most ones for `:py:func:`file.managed `. - - name - The path the certificate should be present at. - - days_remaining - The certificate will be recreated once the remaining certificate validity - period is less than this number of days. - Defaults to ``90`` (until v3009) or ``7`` (from v3009 onwards). - - ca_server - Request a remotely signed certificate from ca_server. For this to - work, a ``signing_policy`` must be specified, and that same policy - must be configured on the ca_server. Also, the Salt master must - permit peers to call the ``x509.sign_remote_certificate`` function. - See the :ref:`execution module docs ` for details. - - signing_policy - The name of a configured signing policy. Parameters specified in there - are hardcoded and cannot be overridden. This is required for remote signing, - otherwise optional. - - encoding - Specify the encoding of the resulting certificate. It can be serialized - as a ``pem`` (or ``pkcs7_pem``) text file or in several binary formats - (``der``, ``pkcs7_der``, ``pkcs12``). Defaults to ``pem``. - - append_certs - A list of additional certificates to append to the new one, e.g. to create a CA chain. - - .. note:: - - Mind that when ``der`` encoding is in use, appending certificatees is prohibited. - - copypath - Create a copy of the issued certificate in PEM format in this directory. - The file will be named ``.crt`` if prepend_cn is false. - - prepend_cn - When ``copypath`` is set, prepend the common name of the certificate to - the file name like so: ``-.crt``. Defaults to false. - - digest - The hashing algorithm to use for the signature. Valid values are: - sha1, sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, - sha3_256, sha3_384, sha3_512. Defaults to ``sha256``. - This will be ignored for ``ed25519`` and ``ed448`` key types. - - signing_private_key - The private key corresponding to the public key in ``signing_cert``. Required. - - signing_private_key_passphrase - If ``signing_private_key`` is encrypted, the passphrase to decrypt it. - - signing_cert - The CA certificate to be used for signing the issued certificate. - - public_key - The public key the certificate should be issued for. Other ways of passing - the required information are ``private_key`` and ``csr``. If neither are set, - the public key of the ``signing_private_key`` will be included, i.e. - a self-signed certificate is generated. - - private_key - The private key corresponding to the public key the certificate should - be issued for. This is one way of specifying the public key that will - be included in the certificate, the other ones being ``public_key`` and ``csr``. - - private_key_passphrase - If ``private_key`` is specified and encrypted, the passphrase to decrypt it. - - csr - A certificate signing request to use as a base for generating the certificate. - The following information will be respected, depending on configuration: - - * public key - * extensions, if not otherwise specified (arguments, signing_policy) - - subject - The subject's distinguished name embedded in the certificate. This is one way of - passing this information (see ``kwargs`` below for the other). - This argument will be preferred and allows to control the order of RDNs in the DN - as well as to embed RDNs with multiple attributes. - This can be specified as a RFC4514-encoded string (``CN=example.com,O=Example Inc,C=US``, - mind that the rendered order is reversed from what is embedded), a list - of RDNs encoded as in RFC4514 (``["C=US", "O=Example Inc", "CN=example.com"]``) - or a dictionary (``{"CN": "example.com", "C": "US", "O": "Example Inc"}``, - default ordering). - Multiple name attributes per RDN are concatenated with a ``+``. - - .. note:: - - Parsing of RFC4514 strings requires at least cryptography release 37. - - serial_number - A serial number to be embedded in the certificate. If unspecified, will - autogenerate one. This should be an integer, either in decimal or - hexadecimal notation. - - not_before - Set a specific date the certificate should not be valid before. - The format should follow ``%Y-%m-%d %H:%M:%S`` and will be interpreted as GMT/UTC. - Defaults to the time of issuance. - - not_after - Set a specific date the certificate should not be valid after. - The format should follow ``%Y-%m-%d %H:%M:%S`` and will be interpreted as GMT/UTC. - If unspecified, defaults to the current time plus ``days_valid`` days. - - days_valid - If ``not_after`` is unspecified, the number of days from the time of issuance - the certificate should be valid for. - Defaults to ``365`` (until v3009) or ``30`` (from v3009 onwards). - - pkcs12_passphrase - When encoding a certificate as ``pkcs12``, encrypt it with this passphrase. - - .. note:: - - PKCS12 encryption is very weak and `should not be relied on for security `_. - - pkcs12_encryption_compat - OpenSSL 3 and cryptography v37 switched to a much more secure default - encryption for PKCS12, which might be incompatible with some systems. - This forces the legacy encryption. Defaults to False. - - pkcs12_friendlyname - When encoding a certificate as ``pkcs12``, a name for the certificate can be included. - - kwargs - Embedded X.509v3 extensions and the subject's distinguished name can be - controlled via supplemental keyword arguments. See - :py:func:`x509.create_certificate ` - for an overview. - """ - # Deprecation checks vs the old x509 module - if days_valid is None and not_after is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_valid` will change to 30. Please adapt your code accordingly.", - ) - days_valid = 365 - except RuntimeError: - days_valid = 30 - - if days_remaining is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_remaining` will change to 7. Please adapt your code accordingly.", - ) - days_remaining = 90 - except RuntimeError: - days_remaining = 7 - - if "algorithm" in kwargs: - salt.utils.versions.warn_until( - "Potassium", - "`algorithm` has been renamed to `digest`. Please update your code.", - ) - digest = kwargs.pop("algorithm") - kwargs = x509util.ensure_cert_kwargs_compat(kwargs) - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The certificate is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, cert_args = _split_file_kwargs(_filter_state_internal_kwargs(kwargs)) - append_certs = append_certs or [] - if not isinstance(append_certs, list): - append_certs = [append_certs] - - try: - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - if __salt__["file.file_exists"](real_name): - try: - ( - current, - current_encoding, - current_chain, - current_extra, - ) = x509util.load_cert( - real_name, passphrase=pkcs12_passphrase, get_encoding=True - ) - except SaltInvocationError as err: - if "Bad decrypt" in str(err): - changes["pkcs12_passphrase"] = True - elif any( - ( - "Could not deserialize binary data" in str(err), - "Could not load PEM-encoded" in str(err), - ) - ): - replace = True - else: - raise - else: - if encoding != current_encoding: - changes["encoding"] = encoding - elif encoding == "pkcs12" and current_extra.cert.friendly_name != ( - salt.utils.stringutils.to_bytes(pkcs12_friendlyname) - if pkcs12_friendlyname - else None - ): - changes["pkcs12_friendlyname"] = pkcs12_friendlyname - - if ( - current.not_valid_after - < datetime.datetime.utcnow() - + datetime.timedelta(days=days_remaining) - ): - changes["expiration"] = True - - current_chain = current_chain or [] - ca_chain = [x509util.load_cert(x) for x in append_certs] - if not _compare_ca_chain(current_chain, ca_chain): - changes["additional_certs"] = True - - ( - builder, - private_key_loaded, - signing_cert_loaded, - final_kwargs, - ) = _build_cert( - ca_server=ca_server, - signing_policy=signing_policy, - digest=digest, # passed because of signing_policy merging - signing_private_key=signing_private_key, - signing_private_key_passphrase=signing_private_key_passphrase, - signing_cert=signing_cert, - public_key=public_key, - private_key=private_key, - private_key_passphrase=private_key_passphrase, - csr=csr, - subject=subject, - serial_number=serial_number, - not_before=not_before, - not_after=not_after, - days_valid=days_valid, - **cert_args, - ) - - try: - if current.signature_hash_algorithm is not None and not isinstance( - current.signature_hash_algorithm, - type(x509util.get_hashing_algorithm(final_kwargs["digest"])), - ): - # ed25519, ed448 do not use a separate hash for signatures, hence algo is None - changes["digest"] = digest - except UnsupportedAlgorithm: - # this eg happens with sha3 in cryptography < v39 - log.warning( - "Could not determine signature hash algorithm of '%s'. " - "Continuing anyways", - name, - ) - - changes.update( - _compare_cert( - current, - builder, - signing_cert=signing_cert_loaded, - serial_number=serial_number, - not_before=not_before, - not_after=not_after, - ) - ) - else: - changes["created"] = name - - if replace: - changes["replaced"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if current and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The certificate would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - { - "additional_certs", - "encoding", - "pkcs12_friendlyname", - }: - # do not reissue if only metaparameters changed - if encoding == "pkcs12": - cert = __salt__["x509.encode_certificate"]( - current, - append_certs=append_certs, - encoding=encoding, - private_key=private_key_loaded, - pkcs12_passphrase=pkcs12_passphrase, - pkcs12_encryption_compat=pkcs12_encryption_compat, - pkcs12_friendlyname=pkcs12_friendlyname, - ) - else: - cert = __salt__["x509.encode_certificate"]( - current, encoding=encoding, append_certs=append_certs - ) - else: - # request a new certificate otherwise - cert = __salt__["x509.create_certificate"]( - ca_server=ca_server, - signing_policy=signing_policy, - encoding=encoding, - append_certs=append_certs, - pkcs12_passphrase=pkcs12_passphrase, - pkcs12_encryption_compat=pkcs12_encryption_compat, - pkcs12_friendlyname=pkcs12_friendlyname, - digest=digest, - signing_private_key=signing_private_key, - signing_private_key_passphrase=signing_private_key_passphrase, - signing_cert=signing_cert, - public_key=public_key, - private_key=private_key, - private_key_passphrase=private_key_passphrase, - csr=csr, - subject=subject, - serial_number=serial_number, - not_before=not_before, - not_after=not_after, - days_valid=days_valid, - **cert_args, - ) - ret["comment"] = f"The certificate has been {verb}d" - if encoding not in ["pem", "pkcs7_pem"]: - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(cert), file_args.get("backup", "") - ) - - if not changes or encoding in ["pem", "pkcs7_pem"]: - replace = bool(encoding in ["pem", "pkcs7_pem"] and changes) - contents = cert if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def crl_managed( - name, - signing_private_key, - revoked, - days_remaining=None, - signing_cert=None, - signing_private_key_passphrase=None, - include_expired=False, - days_valid=None, - digest="sha256", - encoding="pem", - extensions=None, - **kwargs, -): - """ - Ensure a certificate revocation list is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_crl `, - as well as most ones for `:py:func:`file.managed `. - - name - The path the certificate revocation list should be present at. - - signing_private_key - Your certificate authority's private key. It will be used to sign - the CRL. Required. - - revoked - A list of dicts containing all the certificates to revoke. Each dict - represents one certificate. A dict must contain either the key - ``serial_number`` with the value of the serial number to revoke, or - ``certificate`` with some reference to the certificate to revoke. - - The dict can optionally contain the ``revocation_date`` key. If this - key is omitted, the revocation date will be set to now. It should be a - string in the format ``%Y-%m-%d %H:%M:%S``. - - The dict can also optionally contain the ``not_after`` key. This is - redundant if the ``certificate`` key is included. If the - ``certificate`` key is not included, this can be used for the logic - behind the ``include_expired`` parameter. It should be a string in - the format ``%Y-%m-%d %H:%M:%S``. - - The dict can also optionally contain the ``extensions`` key, which - allows to set CRL entry-specific extensions. The following extensions - are supported: - - certificateIssuer - Identifies the certificate issuer associated with an entry in an - indirect CRL. The format is the same as for subjectAltName. - - CRLReason - Identifies the reason for certificate revocation. - Available choices are ``unspecified``, ``keyCompromise``, ``CACompromise``, - ``affiliationChanged``, ``superseded``, ``cessationOfOperation``, - ``certificateHold``, ``privilegeWithdrawn``, ``aACompromise`` and - ``removeFromCRL``. - - invalidityDate - Provides the date on which the certificate became invalid. - The value should be a string in the same format as ``revocation_date``. - - days_remaining - The certificate revocation list will be recreated once the remaining - CRL validity period is less than this number of days. - Defaults to ``30`` (until v3009) or ``3`` (from v3009 onwards). - Set to 0 to disable automatic renewal without anything changing. - - signing_cert - The CA certificate to be used for signing the issued certificate. - - signing_private_key_passphrase - If ``signing_private_key`` is encrypted, the passphrase to decrypt it. - - include_expired - Also include already expired certificates in the CRL. Defaults to false. - - days_valid - The number of days that the CRL should be valid for. This sets the ``Next Update`` - field in the CRL. Defaults to ``100`` (until v3009) or ``7`` (from v3009 onwards). - - digest - The hashing algorithm to use for the signature. Valid values are: - sha1, sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, - sha3_256, sha3_384, sha3_512. Defaults to ``sha256``. - This will be ignored for ``ed25519`` and ``ed448`` key types. - - encoding - Specify the encoding of the resulting certificate revocation list. - It can be serialized as a ``pem`` text or binary ``der`` file. - Defaults to ``pem``. - - extensions - Add CRL extensions. See :py:func:`x509.create_crl ` - for details. - - .. note:: - - For ``cRLNumber``, in addition the value ``auto`` is supported, which - automatically increases the counter every time a new CRL is issued. - - Example: - - .. code-block:: yaml - - Manage CRL: - x509.crl_managed: - - name: /etc/pki/ca.crl - - signing_private_key: /etc/pki/myca.key - - signing_cert: /etc/pki/myca.crt - - revoked: - - certificate: /etc/pki/certs/badweb.crt - revocation_date: 2022-11-01 00:00:00 - extensions: - CRLReason: keyCompromise - - serial_number: D6:D2:DC:D8:4D:5C:C0:F4 - not_after: 2023-03-14 00:00:00 - revocation_date: 2022-10-25 00:00:00 - extensions: - CRLReason: cessationOfOperation - - extensions: - cRLNumber: auto - """ - if "text" in kwargs: - salt.utils.versions.kwargs_warn_until(["text"], "Potassium") - kwargs.pop("text") - - if days_valid is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_valid` will change to 7. Please adapt your code accordingly.", - ) - days_valid = 100 - except RuntimeError: - days_valid = 7 - - if days_remaining is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_remaining` will change to 3. Please adapt your code accordingly.", - ) - days_remaining = 30 - except RuntimeError: - days_remaining = 3 - - revoked_parsed = [] - for rev in revoked: - parsed = {} - if len(rev) == 1 and isinstance(rev[next(iter(rev))], list): - salt.utils.versions.warn_until( - "Potassium", - "Revoked certificates should be specified as a simple list of dicts.", - ) - for val in rev[next(iter(rev))]: - parsed.update(val) - if "reason" in (parsed or rev): - salt.utils.versions.warn_until( - "Potassium", - "The `reason` parameter for revoked certificates should be specified in extensions:CRLReason.", - ) - salt.utils.dictupdate.set_dict_key_value( - (parsed or rev), "extensions:CRLReason", (parsed or rev).pop("reason") - ) - revoked_parsed.append(parsed or rev) - revoked = revoked_parsed - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The certificate revocation list is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, extra_args = _split_file_kwargs(_filter_state_internal_kwargs(kwargs)) - extensions = extensions or {} - if extra_args: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(extra_args)}") - - try: - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - if __salt__["file.file_exists"](real_name): - try: - current, current_encoding = x509util.load_crl( - real_name, get_encoding=True - ) - except SaltInvocationError as err: - if any( - ( - "Could not load PEM-encoded" in str(err), - "Could not load DER-encoded" in str(err), - ) - ): - replace = True - else: - raise - else: - try: - if current.signature_hash_algorithm is not None and not isinstance( - current.signature_hash_algorithm, - type(x509util.get_hashing_algorithm(digest)), - ): - # ed25519, ed448 do not use a separate hash for signatures, hence algo is None - # although CA certificates should not be using those currently - changes["digest"] = digest - except UnsupportedAlgorithm: - # this eg happens with sha3 digest in cryptography < v39 - log.warning( - "Could not determine signature hash algorithm of '%s'. " - "Continuing anyways", - name, - ) - - if encoding != current_encoding: - changes["encoding"] = encoding - if days_remaining and ( - current.next_update - < datetime.datetime.utcnow() - + datetime.timedelta(days=days_remaining) - ): - changes["expiration"] = True - - # "auto" is a value that is managed in this function and cannot not be compared - crl_auto = extensions.get("cRLNumber") == "auto" - if crl_auto: - extensions.pop("cRLNumber") - - builder, sig_privkey = x509util.build_crl( - signing_private_key, - revoked, - signing_cert=signing_cert, - signing_private_key_passphrase=signing_private_key_passphrase, - include_expired=include_expired, - days_valid=days_valid, - extensions=extensions, - ) - changes.update(_compare_crl(current, builder, sig_privkey.public_key())) - if crl_auto: - # put cRLNumber = auto back if it was set - extensions["cRLNumber"] = "auto" - changes["extensions"]["removed"].pop( - changes["extensions"]["removed"].index("cRLNumber") - ) - if not any(changes["extensions"].values()): - changes.pop("extensions") - else: - changes["created"] = name - - if replace: - changes["replaced"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if current and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The certificate revocation list would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - {"encoding"}: - # do not regenerate if only metaparameters changed - crl = __salt__["x509.encode_crl"](current, encoding=encoding) - else: - # autoincrease cRLNumber counter, if requested - if extensions.get("cRLNumber") == "auto": - try: - extensions["cRLNumber"] = ( - current.extensions.get_extension_for_class( - cx509.CRLNumber - ).value.crl_number - + 1 - ) - except (AttributeError, cx509.ExtensionNotFound): - extensions["cRLNumber"] = 1 - crl = __salt__["x509.create_crl"]( - signing_private_key, - revoked, - signing_cert=signing_cert, - signing_private_key_passphrase=signing_private_key_passphrase, - include_expired=include_expired, - days_valid=days_valid, - digest=digest, - encoding=encoding, - extensions=extensions, - ) - ret["comment"] = f"The certificate revocation list has been {verb}d" - if encoding == "der": - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(crl), file_args.get("backup", "") - ) - - if not changes or encoding == "pem": - replace = bool((encoding == "pem") and changes) - contents = crl if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def csr_managed( - name, - private_key, - private_key_passphrase=None, - digest="sha256", - encoding="pem", - subject=None, - **kwargs, -): - """ - Ensure a certificate signing request is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_csr `, - as well as most ones for :py:func:`file.managed `. - - name - The path the certificate signing request should be present at. - - private_key - The private key corresponding to the public key the certificate should - be issued for. The CSR will be signed by it. Required. - - private_key_passphrase - If ``private_key`` is encrypted, the passphrase to decrypt it. - - digest - The hashing algorithm to use for the signature. Valid values are: - sha1, sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, - sha3_256, sha3_384, sha3_512. Defaults to ``sha256``. - This will be ignored for ``ed25519`` and ``ed448`` key types. - - encoding - Specify the encoding of the resulting certificate revocation list. - It can be serialized as a ``pem`` text or binary ``der`` file. - Defaults to ``pem``. - - kwargs - Embedded X.509v3 extensions and the subject's distinguished name can be - controlled via supplemental keyword arguments. - See :py:func:`x509.create_certificate ` - for an overview. Mind that some extensions are not available for CSR - (``authorityInfoAccess``, ``authorityKeyIdentifier``, - ``issuerAltName``, ``crlDistributionPoints``). - """ - # Deprecation checks vs the old x509 module - if "algorithm" in kwargs: - salt.utils.versions.warn_until( - "Potassium", - "`algorithm` has been renamed to `digest`. Please update your code.", - ) - digest = kwargs.pop("algorithm") - kwargs = x509util.ensure_cert_kwargs_compat(kwargs) - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The certificate signing request is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, csr_args = _split_file_kwargs(_filter_state_internal_kwargs(kwargs)) - - try: - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - if __salt__["file.file_exists"](real_name): - try: - current, current_encoding = x509util.load_csr( - real_name, get_encoding=True - ) - except SaltInvocationError as err: - if any( - ( - "Could not load PEM-encoded" in str(err), - "Could not load DER-encoded" in str(err), - ) - ): - replace = True - else: - raise - except cx509.InvalidVersion: - # by default, the previous x509 modules generated CSR with - # invalid versions, which leads to an exception in cryptography >= v38 - changes["invalid_version"] = True - replace = True - else: - try: - if current.signature_hash_algorithm is not None and not isinstance( - current.signature_hash_algorithm, - type(x509util.get_hashing_algorithm(digest)), - ): - # ed25519, ed448 do not use a separate hash for signatures, hence algo is None - changes["digest"] = digest - except UnsupportedAlgorithm: - # this eg happens with sha3 digest in cryptography < v39 - log.warning( - "Could not determine signature hash algorithm of '%s'. " - "Continuing anyways", - name, - ) - - if encoding != current_encoding: - changes["encoding"] = encoding - - builder, privkey = x509util.build_csr( - private_key, - private_key_passphrase=private_key_passphrase, - subject=subject, - **csr_args, - ) - if not x509util.is_pair(current.public_key(), privkey): - changes["private_key"] = True - changes.update(_compare_csr(current, builder)) - else: - changes["created"] = name - - if replace: - changes["replaced"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if current and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The certificate signing request would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - {"encoding"}: - # do not regenerate if only metaparameters changed - csr = __salt__["x509.encode_csr"](current, encoding=encoding) - else: - csr = __salt__["x509.create_csr"]( - private_key, - private_key_passphrase=private_key_passphrase, - digest=digest, - encoding=encoding, - subject=subject, - **csr_args, - ) - ret["comment"] = f"The certificate signing request has been {verb}d" - if encoding == "der": - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(csr), file_args.get("backup", "") - ) - if not changes or encoding == "pem": - replace = bool((encoding == "pem") and changes) - contents = csr if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def pem_managed(name, text, **kwargs): - """ - Manage the contents of a PEM file directly with the content in text, - ensuring correct formatting. - - name - The path to the file to manage. - - text - The PEM-formatted text to write. - - kwargs - Most arguments supported by :py:func:`file.managed ` are passed through. - """ - file_args, extra_args = _split_file_kwargs(kwargs) - if extra_args: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(extra_args)}") - - try: - file_args["contents"] = __salt__["x509.get_pem_entry"](text=text) - except (CommandExecutionError, SaltInvocationError) as err: - return {"name": name, "result": False, "comment": str(err), "changes": {}} - return _file_managed(name, **file_args) - - -def private_key_managed( - name, - algo="rsa", - keysize=None, - passphrase=None, - encoding="pem", - new=False, - overwrite=False, - pkcs12_encryption_compat=False, - **kwargs, -): - """ - Ensure a private key is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_private_key `, - as well as most ones for :py:func:`file.managed `. - - .. note:: - - If ``mode`` is unspecified, it will default to ``0400``. - - name - The path the private key should be present at. - - algo - The digital signature scheme the private key should be based on. - Available: ``rsa``, ``ec``, ``ed25519``, ``ed448``. Defaults to ``rsa``. - - keysize - For ``rsa``, specifies the bitlength of the private key (2048, 3072, 4096). - For ``ec``, specifies the NIST curve to use (256, 384, 521). - Irrelevant for Edwards-curve schemes (`ed25519``, ``ed448``). - Defaults to 2048 for RSA and 256 for EC. - - passphrase - If this is specified, the private key will be encrypted using this - passphrase. The encryption algorithm cannot be selected, it will be - determined automatically as the best available one. - - encoding - Specify the encoding of the resulting private key. It can be serialized - as a ``pem`` text, binary ``der`` or ``pkcs12`` file. - Defaults to ``pem``. - - new - Always create a new key. Defaults to false. - Combining new with :mod:`prereq ` - can allow key rotation whenever a new certificate is generated. - - overwrite - Overwrite an existing private key if the provided passphrase cannot decrypt it. - Defaults to false. - - pkcs12_encryption_compat - Some operating systems are incompatible with the encryption defaults - for PKCS12 used since OpenSSL v3. This switch triggers a fallback to - ``PBESv1SHA1And3KeyTripleDESCBC``. - Please consider the `notes on PKCS12 encryption `_. - - Example: - - The Jinja templating in this example ensures a new private key is generated - if the file does not exist and whenever the associated certificate - is to be renewed. - - .. code-block:: jinja - - Manage www private key: - x509.private_key_managed: - - name: /etc/pki/www.key - - keysize: 4096 - - new: true - {%- if salt["file.file_exists"]("/etc/pki/www.key") %} - - prereq: - - x509: /etc/pki/www.crt - {%- endif %} - """ - # Deprecation checks vs the old x509 module - if "bits" in kwargs: - salt.utils.versions.warn_until( - "Potassium", - "`bits` has been renamed to `keysize`. Please update your code.", - ) - keysize = kwargs.pop("bits") - - ignored_params = {"cipher", "verbose", "text"}.intersection( - kwargs - ) # path, overwrite - if ignored_params: - salt.utils.versions.kwargs_warn_until(ignored_params, "Potassium") - for x in ignored_params: - kwargs.pop(x) - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The private key is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, extra_args = _split_file_kwargs(kwargs) - - if extra_args: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(extra_args)}") - - if not file_args.get("mode"): - # ensure secure defaults - file_args["mode"] = "0400" - - try: - if keysize and algo in ["ed25519", "ed448"]: - raise SaltInvocationError(f"keysize is an invalid parameter for {algo}") - - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - file_exists = __salt__["file.file_exists"](real_name) - - if file_exists and not new: - try: - current, current_encoding, _ = x509util.load_privkey( - real_name, passphrase=passphrase, get_encoding=True - ) - except SaltInvocationError as err: - if "Bad decrypt" in str(err): - if not overwrite: - raise CommandExecutionError( - "The provided passphrase cannot decrypt the private key. " - "Pass overwrite: true to force regeneration" - ) from err - changes["passphrase"] = True - elif any( - ( - "Could not deserialize binary data" in str(err), - "Could not load DER-encoded" in str(err), - "Could not load PEM-encoded" in str(err), - ) - ): - if not overwrite: - raise CommandExecutionError( - "The existing file does not seem to be a private key " - "formatted as DER, PEM or embedded in PKCS12. " - "Pass overwrite: true to force regeneration" - ) from err - replace = True - elif "Private key is unencrypted" in str(err): - changes["passphrase"] = True - current, current_encoding, _ = x509util.load_privkey( - real_name, passphrase=None, get_encoding=True - ) - elif "Private key is encrypted" in str(err) and not passphrase: - if not overwrite: - raise CommandExecutionError( - "The existing file is encrypted. Pass overwrite: true " - "to force regeneration without passphrase" - ) from err - changes["passphrase"] = True - else: - raise - if current: - key_type = x509util.get_key_type(current) - check_keysize = keysize - if check_keysize is None: - if algo == "rsa": - check_keysize = 2048 - elif algo == "ec": - check_keysize = 256 - if any( - ( - (algo == "rsa" and not key_type == x509util.KEY_TYPE.RSA), - (algo == "ec" and not key_type == x509util.KEY_TYPE.EC), - (algo == "ed25519" and not key_type == x509util.KEY_TYPE.ED25519), - (algo == "ed448" and not key_type == x509util.KEY_TYPE.ED448), - ) - ): - changes["algo"] = algo - if ( - "algo" not in changes - and algo in ("rsa", "ec") - and current.key_size != check_keysize - ): - changes["keysize"] = keysize - if encoding != current_encoding: - changes["encoding"] = encoding - elif file_exists and new: - changes["replaced"] = name - else: - changes["created"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if file_exists and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The private key would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - {"encoding", "passphrase"}: - # do not regenerate if only metaparameters changed - pk = __salt__["x509.encode_private_key"]( - current, passphrase=passphrase, encoding=encoding - ) - else: - pk = __salt__["x509.create_private_key"]( - algo=algo, - keysize=keysize, - passphrase=passphrase, - encoding=encoding, - pkcs12_encryption_compat=pkcs12_encryption_compat, - ) - ret["comment"] = f"The private key has been {verb}d" - if encoding != "pem": - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(pk), file_args.get("backup", "") - ) - - if not changes or encoding == "pem": - replace = bool((encoding == "pem") and changes) - contents = pk if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def _filter_state_internal_kwargs(kwargs): - # check_cmd is a valid argument to file.managed - ignore = set(_STATE_INTERNAL_KEYWORDS) - {"check_cmd"} - return {k: v for k, v in kwargs.items() if k not in ignore} - - -def _split_file_kwargs(kwargs): - valid_file_args = [ - "user", - "group", - "mode", - "attrs", - "makedirs", - "dir_mode", - "backup", - "create", - "follow_symlinks", - "check_cmd", - "tmp_dir", - "tmp_ext", - "selinux", - "encoding", - "encoding_errors", - "win_owner", - "win_perms", - "win_deny_perms", - "win_inheritance", - "win_perms_reset", - ] - file_args = {"show_changes": False} - extra_args = {} - for k, v in kwargs.items(): - if k in valid_file_args: - file_args[k] = v - else: - extra_args[k] = v - return file_args, extra_args - - -def _add_sub_state_run(ret, sub): - sub["low"] = { - "name": ret["name"], - "state": "file", - "__id__": __low__["__id__"], - "fun": "managed", - } - if "sub_state_run" not in ret: - ret["sub_state_run"] = [] - ret["sub_state_run"].append(sub) - - -def _file_managed(name, test=None, **kwargs): - if test not in [None, True]: - raise SaltInvocationError("test param can only be None or True") - # work around https://github.com/saltstack/salt/issues/62590 - test = test or __opts__["test"] - file_managed = __states__["file.managed"] - if test: - # calls via __salt__["state.single"](..., test=test) - # can overwrite __opts__["test"] permanently. Workaround: - opts = __opts__ - if not __opts__["test"]: - opts = copy.copy(__opts__) - opts["test"] = test - with salt.utils.context.func_globals_inject(file_managed, __opts__=opts): - # The file execution module accesses __opts__["test"] as well - with salt.utils.context.func_globals_inject( - __salt__["file.check_perms"], __opts__=opts - ): - with salt.utils.context.func_globals_inject( - __salt__["file.manage_file"], __opts__=opts - ): - return file_managed(name, **kwargs) - return file_managed(name, **kwargs) - - -def _check_file_ret(fret, ret, current): - if fret["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = f"Could not {'create' if not current else 'update'} file, see file.managed output" - ret["changes"] = {} - return False - return True - - -def _build_cert( - ca_server=None, signing_policy=None, signing_private_key=None, **kwargs -): - final_kwargs = copy.deepcopy(kwargs) - x509util.merge_signing_policy( - __salt__["x509.get_signing_policy"](signing_policy, ca_server=ca_server), - final_kwargs, - ) - - builder, _, private_key_loaded, signing_cert = x509util.build_crt( - signing_private_key, - skip_load_signing_private_key=ca_server is not None, - **final_kwargs, - ) - return builder, private_key_loaded, signing_cert, final_kwargs - - -def _compare_cert(current, builder, signing_cert, serial_number, not_before, not_after): - changes = {} - - if ( - serial_number is not None - and _getattr_safe(builder, "_serial_number") != current.serial_number - ): - changes["serial_number"] = serial_number - - if not x509util.match_pubkey( - _getattr_safe(builder, "_public_key"), current.public_key() - ): - changes["private_key"] = True - - if signing_cert and not x509util.verify_signature( - current, signing_cert.public_key() - ): - changes["signing_private_key"] = True - - if _getattr_safe(builder, "_subject_name") != current.subject: - changes["subject_name"] = _getattr_safe( - builder, "_subject_name" - ).rfc4514_string() - - if _getattr_safe(builder, "_issuer_name") != current.issuer: - changes["issuer_name"] = _getattr_safe(builder, "_issuer_name").rfc4514_string() - - ext_changes = _compare_exts(current, builder) - if any(ext_changes.values()): - changes["extensions"] = ext_changes - return changes - - -def _compare_csr(current, builder): - changes = {} - - # if _getattr_safe(builder, "_subject_name") != current.subject: - if not _compareattr_safe(builder, "_subject_name", current.subject): - changes["subject_name"] = _getattr_safe( - builder, "_subject_name" - ).rfc4514_string() - - ext_changes = _compare_exts(current, builder) - if any(ext_changes.values()): - changes["extensions"] = ext_changes - return changes - - -def _compare_crl(current, builder, sig_pubkey): - # these are necessary because the classes do not have the required method - def _get_revoked_certificate_by_serial_number(revoked, serial): - try: - return [x for x in revoked if x.serial_number == serial][0] - except IndexError: - return None - - def _get_extension_for_oid(extensions, oid): - try: - return [x for x in extensions if x.oid == oid][0] - except IndexError: - return None - - changes = {} - - if _getattr_safe(builder, "_issuer_name") != current.issuer: - changes["issuer_name"] = _getattr_safe(builder, "_issuer_name").rfc4514_string() - if not current.is_signature_valid(sig_pubkey): - changes["public_key"] = True - - rev_changes = {"added": [], "changed": [], "removed": []} - revoked = _getattr_safe(builder, "_revoked_certificates") - for rev in revoked: - cur = current.get_revoked_certificate_by_serial_number(rev.serial_number) - if cur is None: - # certificate was not revoked before - rev_changes["added"].append(x509util.dec2hex(rev.serial_number)) - continue - - for ext in rev.extensions: - cur_ext = _get_extension_for_oid(cur.extensions, ext.oid) - # revoked certificate's extensions have changed (added/changed) - if any( - ( - cur_ext is None, - cur_ext.critical != ext.critical, - cur_ext.value != ext.value, - ) - ): - rev_changes["changed"].append(x509util.dec2hex(rev.serial_number)) - - for cur_ext in cur.extensions: - if _get_extension_for_oid(rev.extensions, cur_ext.oid) is None: - # an extension was removed from from the revoked certificate - rev_changes["changed"].append(x509util.dec2hex(rev.serial_number)) - - for rev in current: - # certificate was removed from the CRL, probably because it was outdated anyways - if ( - _get_revoked_certificate_by_serial_number(revoked, rev.serial_number) - is None - ): - rev_changes["removed"].append(x509util.dec2hex(rev.serial_number)) - - if any(rev_changes.values()): - changes["revocations"] = rev_changes - - ext_changes = _compare_exts(current, builder) - if any(ext_changes.values()): - changes["extensions"] = ext_changes - return changes - - -def _compare_exts(current, builder): - def getextname(ext): - try: - return ext.oid._name - except AttributeError: - return ext.oid.dotted_string - - added = [] - changed = [] - removed = [] - builder_extensions = cx509.Extensions(_getattr_safe(builder, "_extensions")) - - # iter is unnecessary, but avoids a pylint < 2.13.6 crash - for ext in iter(builder_extensions): - try: - cur_ext = current.extensions.get_extension_for_oid(ext.value.oid) - if cur_ext.critical != ext.critical or cur_ext.value != ext.value: - changed.append(getextname(ext)) - except cx509.ExtensionNotFound: - added.append(getextname(ext)) - - for ext in current.extensions: - try: - builder_extensions.get_extension_for_oid(ext.value.oid) - except cx509.ExtensionNotFound: - removed.append(getextname(ext)) - - return {"added": added, "changed": changed, "removed": removed} - - -def _compare_ca_chain(current, new): - if not len(current) == len(new): - return False - for i, new_cert in enumerate(new): - if new_cert.fingerprint(hashes.SHA256()) != current[i].fingerprint( - hashes.SHA256() - ): - return False - return True - - -def _getattr_safe(obj, attr): - try: - return getattr(obj, attr) - except AttributeError as err: - # Since we cannot get the certificate object without signing, - # we need to compare attributes marked as internal. At least - # convert possible exceptions into some description. - raise CommandExecutionError( - f"Could not get attribute {attr} from {obj.__class__.__name__}. " - "Did the internal API of cryptography change?" - ) from err - - -def _compareattr_safe(obj, attr, comp): - try: - return getattr(obj, attr) == comp - except AttributeError: - return False - - -def _safe_atomic_write(dst, data, backup): - """ - Create a temporary file with only user r/w perms and atomically - copy it to the destination, honoring ``backup``. - """ - tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX) - with salt.utils.files.fopen(tmp, "wb") as tmp_: - tmp_.write(data) - salt.utils.files.copyfile( - tmp, dst, __salt__["config.backup_mode"](backup), __opts__["cachedir"] - ) - salt.utils.files.safe_rm(tmp) diff --git a/setup/so-functions b/setup/so-functions index b97d4bb52..82dd9d375 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1960,11 +1960,8 @@ salt_install_module_deps() { } salt_patch_x509_v2() { - # this can be removed when https://github.com/saltstack/salt/issues/64195 is resolved - if [ $SALTVERSION == "3006.1" ]; then - info "Salt version 3006.1 found. Patching /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py" - \cp -v ./files/patch/states/x509_v2.py /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py - fi + # this can be removed when https://github.com/saltstack/salt/issues/66929 is resolved + logCmd "salt-call state.apply salt.patch.x509_v2 --local --file-root=../salt/" } # Create an secrets pillar so that passwords survive re-install From 667e66bbef008719913d9789b08078c96f7ad01e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 26 Mar 2025 13:56:49 -0400 Subject: [PATCH 31/31] rename mine update and highstate state --- salt/mine/update.sls | 24 -------------------- salt/salt/master/mine_update_highstate.sls | 26 ++++++++++++++++++++++ salt/top.sls | 2 +- 3 files changed, 27 insertions(+), 25 deletions(-) delete mode 100644 salt/mine/update.sls create mode 100644 salt/salt/master/mine_update_highstate.sls diff --git a/salt/mine/update.sls b/salt/mine/update.sls deleted file mode 100644 index 03ea76c9e..000000000 --- a/salt/mine/update.sls +++ /dev/null @@ -1,24 +0,0 @@ -# This state sends an event to the salt-master event bus -# The event will be caught by the reactor and trigger the mine_update orchestration - -{# may be able to use this method if we can figure out multi state run failure - https://github.com/saltstack/salt/issues/66929 #} -# Get the minion ID from the pillar -{% set MINION_ID = grains.id %} - -# Run mine.update on all minions -mine.update.update_mine_all_minions: - salt.function: - - name: mine.update - - tgt: '*' - - batch: 50 - - retry: - attempts: 3 - interval: 1 - -# Run highstate on the original minion -# we can use concurrent on this highstate because no other highstate would be running when this is called -mine.update.run_highstate_on_{{ MINION_ID }}: - salt.state: - - tgt: {{ MINION_ID }} - - highstate: True - - concurrent: True diff --git a/salt/salt/master/mine_update_highstate.sls b/salt/salt/master/mine_update_highstate.sls new file mode 100644 index 000000000..874e6c65b --- /dev/null +++ b/salt/salt/master/mine_update_highstate.sls @@ -0,0 +1,26 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# This state should only be run on managers and should never be run manually + +{% set MINION_ID = grains.id %} + +# Run mine.update on all minions +salt.master.mine_update_highstate.update_mine_all_minions: + salt.function: + - name: mine.update + - tgt: '*' + - batch: 50 + - retry: + attempts: 3 + interval: 1 + +# Run highstate on the original minion +# we can use concurrent on this highstate because no other highstate would be running when this is called +salt.master.mine_update_highstate.run_highstate_on_{{ MINION_ID }}: + salt.state: + - tgt: {{ MINION_ID }} + - highstate: True + - concurrent: True diff --git a/salt/top.sls b/salt/top.sls index 4a6e8e010..ee364b81b 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -20,7 +20,7 @@ base: 'I@node_data:False and ( *_manager* or *_eval or *_import or *_standalone )': - match: compound - salt.minion - - mine.update + - salt.master.mine_update_highstate 'not G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound