diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 693c48505..de1f7a48c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -440,33 +440,7 @@ preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." - [[ "$INSTALLEDVERSION" == 2.4.2 ]] && up_to_2.4.3 - [[ "$INSTALLEDVERSION" == 2.4.3 ]] && up_to_2.4.4 - [[ "$INSTALLEDVERSION" == 2.4.4 ]] && up_to_2.4.5 - [[ "$INSTALLEDVERSION" == 2.4.5 ]] && up_to_2.4.10 - [[ "$INSTALLEDVERSION" == 2.4.10 ]] && up_to_2.4.20 - [[ "$INSTALLEDVERSION" == 2.4.20 ]] && up_to_2.4.30 - [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40 - [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50 - [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 - [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 - [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 - [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 - [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100 - [[ "$INSTALLEDVERSION" == 2.4.100 ]] && up_to_2.4.110 - [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.111 - [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120 - [[ "$INSTALLEDVERSION" == 2.4.120 ]] && up_to_2.4.130 - [[ "$INSTALLEDVERSION" == 2.4.130 ]] && up_to_2.4.140 - [[ "$INSTALLEDVERSION" == 2.4.140 ]] && up_to_2.4.141 - [[ "$INSTALLEDVERSION" == 2.4.141 ]] && up_to_2.4.150 - [[ "$INSTALLEDVERSION" == 2.4.150 ]] && up_to_2.4.160 - [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170 - [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180 - [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190 - [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200 - [[ "$INSTALLEDVERSION" == 2.4.200 ]] && up_to_2.4.201 - [[ "$INSTALLEDVERSION" == 2.4.201 ]] && up_to_2.4.210 + [[ "$INSTALLEDVERSION" == 2.4.2 ]] && up_to_2.4.3 true } @@ -475,32 +449,6 @@ postupgrade_changes() { echo "Running post upgrade processes." [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3 - [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 - [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 - [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 - [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20 - [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30 - [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40 - [[ "$POSTVERSION" == 2.4.40 ]] && post_to_2.4.50 - [[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60 - [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 - [[ "$POSTVERSION" == 2.4.70 ]] && post_to_2.4.80 - [[ "$POSTVERSION" == 2.4.80 ]] && post_to_2.4.90 - [[ "$POSTVERSION" == 2.4.90 ]] && post_to_2.4.100 - [[ "$POSTVERSION" == 2.4.100 ]] && post_to_2.4.110 - [[ "$POSTVERSION" == 2.4.110 ]] && post_to_2.4.111 - [[ "$POSTVERSION" == 2.4.111 ]] && post_to_2.4.120 - [[ "$POSTVERSION" == 2.4.120 ]] && post_to_2.4.130 - [[ "$POSTVERSION" == 2.4.130 ]] && post_to_2.4.140 - [[ "$POSTVERSION" == 2.4.140 ]] && post_to_2.4.141 - [[ "$POSTVERSION" == 2.4.141 ]] && post_to_2.4.150 - [[ "$POSTVERSION" == 2.4.150 ]] && post_to_2.4.160 - [[ "$POSTVERSION" == 2.4.160 ]] && post_to_2.4.170 - [[ "$POSTVERSION" == 2.4.170 ]] && post_to_2.4.180 - [[ "$POSTVERSION" == 2.4.180 ]] && post_to_2.4.190 - [[ "$POSTVERSION" == 2.4.190 ]] && post_to_2.4.200 - [[ "$POSTVERSION" == 2.4.200 ]] && post_to_2.4.201 - [[ "$POSTVERSION" == 2.4.201 ]] && post_to_2.4.210 true } @@ -509,216 +457,11 @@ post_to_2.4.3() { POSTVERSION=2.4.3 } -post_to_2.4.4() { - echo "Nothing to apply" - POSTVERSION=2.4.4 +up_to_2.4.3() { + echo "Nothing to do for 2.4.3" + + INSTALLEDVERSION=2.4.3 } - -post_to_2.4.5() { - echo "Nothing to apply" - POSTVERSION=2.4.5 -} - -post_to_2.4.10() { - echo "Updating Elastic Fleet ES URLs...." - /sbin/so-elastic-fleet-es-url-update --force - POSTVERSION=2.4.10 -} - -post_to_2.4.20() { - echo "Pruning unused docker volumes on all nodes - This process will run in the background." - salt --async \* cmd.run "docker volume prune -f" - POSTVERSION=2.4.20 -} - -post_to_2.4.30() { - # there is an occasional error with this state: pki_public_ca_crt: TypeError: list indices must be integers or slices, not str - set +e - salt-call state.apply ca queue=True - set -e - stop_salt_minion - mv /etc/pki/managerssl.crt /etc/pki/managerssl.crt.old - mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old - systemctl_func "start" "salt-minion" - salt-call state.apply nginx queue=True - enable_highstate - POSTVERSION=2.4.30 -} - -post_to_2.4.40() { - echo "Nothing to apply" - POSTVERSION=2.4.40 -} - -post_to_2.4.50() { - echo "Nothing to apply" - POSTVERSION=2.4.50 -} - -post_to_2.4.60() { - echo "Nothing to apply" - POSTVERSION=2.4.60 -} - -post_to_2.4.70() { - printf "\nRemoving idh.services from any existing IDH node pillar files\n" - for file in /opt/so/saltstack/local/pillar/minions/*.sls; do - if [[ $file =~ "_idh.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then - echo "Removing idh.services from: $file" - so-yaml.py remove "$file" idh.services - fi - done - POSTVERSION=2.4.70 -} - -post_to_2.4.80() { - echo -e "\nChecking if update to Elastic Fleet output policy is required\n" - so-kafka-fleet-output-policy - POSTVERSION=2.4.80 -} - -post_to_2.4.90() { - disable_logstash_heavynodes - POSTVERSION=2.4.90 -} - -post_to_2.4.100() { - echo "Nothing to apply" - POSTVERSION=2.4.100 -} - -post_to_2.4.110() { - echo "Nothing to apply" - POSTVERSION=2.4.110 -} - -post_to_2.4.111() { - echo "Nothing to apply" - POSTVERSION=2.4.111 -} - -post_to_2.4.120() { - update_elasticsearch_index_settings - - # Manually rollover suricata alerts index to ensure data_stream.dataset expected mapping is set to 'suricata' - rollover_index "logs-suricata.alerts-so" - - POSTVERSION=2.4.120 -} - -post_to_2.4.130() { - # Optional integrations are loaded AFTER initial successful load of core ES templates (/opt/so/state/estemplates.txt) - # Dynamic templates are created in elasticsearch.enabled for every optional integration based on output of so-elastic-fleet-optional-integrations-load script - echo "Ensuring Elasticsearch templates are up to date after updating package registry" - salt-call state.apply elasticsearch queue=True - - # Update kibana default space - salt-call state.apply kibana.config queue=True - echo "Updating Kibana default space" - /usr/sbin/so-kibana-space-defaults - - POSTVERSION=2.4.130 -} - -post_to_2.4.140() { - echo "Nothing to apply" - POSTVERSION=2.4.140 -} - -post_to_2.4.141() { - echo "Nothing to apply" - POSTVERSION=2.4.141 -} - -post_to_2.4.150() { - echo "Nothing to apply" - POSTVERSION=2.4.150 -} - -post_to_2.4.160() { - echo "Nothing to apply" - POSTVERSION=2.4.160 -} - -post_to_2.4.170() { - # Update kibana default space - salt-call state.apply kibana.config queue=True - echo "Updating Kibana default space" - /usr/sbin/so-kibana-space-defaults - - POSTVERSION=2.4.170 -} - -post_to_2.4.180() { - # Force update to Kafka output policy - /usr/sbin/so-kafka-fleet-output-policy --force - - POSTVERSION=2.4.180 -} - -post_to_2.4.190() { - # Only need to update import / eval nodes - if [[ "$MINION_ROLE" == "import" ]] || [[ "$MINION_ROLE" == "eval" ]]; then - update_import_fleet_output - fi - - # Check if expected default policy is logstash (global.pipeline is REDIS or "") - pipeline=$(lookup_pillar "pipeline" "global") - if [[ -z "$pipeline" ]] || [[ "$pipeline" == "REDIS" ]]; then - # Check if this grid is currently affected by corrupt fleet output policy - if elastic-agent status | grep "config: key file not configured" > /dev/null 2>&1; then - echo "Elastic Agent shows an ssl error connecting to logstash output. Updating output policy..." - update_default_logstash_output - fi - fi - # Apply new elasticsearch.server index template - rollover_index "logs-elasticsearch.server-default" - - POSTVERSION=2.4.190 -} - -post_to_2.4.200() { - echo "Initiating Suricata idstools migration..." - suricata_idstools_removal_post - - POSTVERSION=2.4.200 -} - -post_to_2.4.201() { - echo "Nothing to apply" - POSTVERSION=2.4.201 -} - -post_to_2.4.210() { - echo "Rolling over Kratos index to apply new index template" - - rollover_index "logs-kratos-so" - - disable_redis_heavynodes - - initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat" - - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers - - # migrate elasticsearch:managed_integrations pillar to manager:managed_integrations - if managed_integrations=$(/usr/sbin/so-yaml.py get /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.managed_integrations 2>/dev/null); then - local managed_integrations_old_pillar="/tmp/elasticsearch-managed_integrations.yaml" - - echo "Migrating managed_integrations pillar" - echo -e "$managed_integrations" > "$managed_integrations_old_pillar" - - /usr/sbin/so-yaml.py add /opt/so/saltstack/local/pillar/manager/soc_manager.sls manager.managed_integrations file:$managed_integrations_old_pillar > /dev/null 2>&1 - - /usr/sbin/so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.managed_integrations - fi - - # Remove so-rule-update script left behind by the idstools removal in 2.4.200 - rm -f /usr/sbin/so-rule-update - - POSTVERSION=2.4.210 -} - repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -769,246 +512,6 @@ stop_salt_minion() { set -e } - -up_to_2.4.3() { - echo "Nothing to do for 2.4.3" - - INSTALLEDVERSION=2.4.3 -} - -up_to_2.4.4() { - echo "Nothing to do for 2.4.4" - - INSTALLEDVERSION=2.4.4 -} - -up_to_2.4.5() { - echo "Nothing to do for 2.4.5" - - INSTALLEDVERSION=2.4.5 -} - -up_to_2.4.10() { - echo "Nothing to do for 2.4.10" - - INSTALLEDVERSION=2.4.10 -} - -up_to_2.4.20() { - echo "Nothing to do for 2.4.20" - - INSTALLEDVERSION=2.4.20 -} - -up_to_2.4.30() { - echo "Nothing to do for 2.4.30" - - INSTALLEDVERSION=2.4.30 -} - -up_to_2.4.40() { - echo "Removing old ATT&CK Navigator Layers..." - rm -f /opt/so/conf/navigator/layers/enterprise-attack.json - rm -f /opt/so/conf/navigator/layers/nav_layer_playbook.json - - INSTALLEDVERSION=2.4.40 -} - -up_to_2.4.50() { - echo "Creating additional pillars.." - mkdir -p /opt/so/saltstack/local/pillar/stig/ - mkdir -p /opt/so/saltstack/local/salt/stig/ - chown socore:socore /opt/so/saltstack/local/salt/stig/ - touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls - touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls - - # the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots - # put new so-yaml in place - echo "Updating so-yaml" - \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" - \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ - echo "Creating a backup of the salt-master config." - # INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version - # so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until - # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup - cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak" - echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" - so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules/nids - echo "Moving Suricata rules" - mkdir /opt/so/rules/nids/suri - chown socore:socore /opt/so/rules/nids/suri - mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/. - - echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml" - so-yaml.py append /etc/salt/master file_roots.base /nsm/elastic-fleet/artifacts - - INSTALLEDVERSION=2.4.50 -} - -up_to_2.4.60() { - echo "Creating directory to store Suricata classification.config" - mkdir -vp /opt/so/saltstack/local/salt/suricata/classification - chown socore:socore /opt/so/saltstack/local/salt/suricata/classification - - INSTALLEDVERSION=2.4.60 -} - -up_to_2.4.70() { - playbook_migration - suricata_idstools_migration - toggle_telemetry - add_detection_test_pillars - - INSTALLEDVERSION=2.4.70 -} - -up_to_2.4.80() { - phases_pillar_2_4_80 - # Kafka configuration changes - - # Global pipeline changes to REDIS or KAFKA - echo "Removing global.pipeline pillar configuration" - sed -i '/pipeline:/d' /opt/so/saltstack/local/pillar/global/soc_global.sls - # Kafka pillars - mkdir -p /opt/so/saltstack/local/pillar/kafka - touch /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - touch /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls - echo 'kafka: ' > /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - kafka_cluster_id=$(get_random_value 22) - echo ' cluster_id: '$kafka_cluster_id >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - kafkapass=$(get_random_value) - echo ' password: '$kafkapass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - - INSTALLEDVERSION=2.4.80 -} - -up_to_2.4.90() { - kafkatrust=$(get_random_value) - # rearranging the kafka pillar to reduce clutter in SOC UI - kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password) - kafkatrimpass=$(echo "$kafkasavedpass" | sed -n '1 p' ) - echo "Making changes to the Kafka pillar layout" - so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password - so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" - so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" - - INSTALLEDVERSION=2.4.90 -} - -up_to_2.4.100() { - echo "Nothing to do for 2.4.100" - - INSTALLEDVERSION=2.4.100 -} - -up_to_2.4.110() { - echo "Nothing to do for 2.4.110" - - INSTALLEDVERSION=2.4.110 -} - -up_to_2.4.111() { - echo "Nothing to do for 2.4.111" - - INSTALLEDVERSION=2.4.111 -} - -up_to_2.4.120() { - add_hydra_pillars - - # this is needed for the new versionlock state - mkdir -p /opt/so/saltstack/local/pillar/versionlock - touch /opt/so/saltstack/local/pillar/versionlock/adv_versionlock.sls /opt/so/saltstack/local/pillar/versionlock/soc_versionlock.sls - - - INSTALLEDVERSION=2.4.120 -} - -up_to_2.4.130() { - # Remove any old Elastic Defend config files - rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json - - # Ensure override exists to allow nmcli access to other devices - touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf - - INSTALLEDVERSION=2.4.130 -} - -up_to_2.4.140() { - echo "Nothing to do for 2.4.140" - - INSTALLEDVERSION=2.4.140 -} - -up_to_2.4.141() { - echo "Nothing to do for 2.4.141" - - INSTALLEDVERSION=2.4.141 -} - -up_to_2.4.150() { - echo "If the Detection indices exists, update the refresh_interval" - so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' - - INSTALLEDVERSION=2.4.150 -} - -up_to_2.4.160() { - echo "Nothing to do for 2.4.160" - - INSTALLEDVERSION=2.4.160 -} - -up_to_2.4.170() { - echo "Creating pillar files for virtualization feature" - - states=("hypervisor" "vm" "libvirt") - - # Create pillar files for each state - for state in "${states[@]}"; do - mkdir -p /opt/so/saltstack/local/pillar/$state - touch /opt/so/saltstack/local/pillar/$state/adv_$state.sls /opt/so/saltstack/local/pillar/$state/soc_$state.sls - done - - - INSTALLEDVERSION=2.4.170 -} - -up_to_2.4.180() { - echo "Nothing to do for 2.4.180" - INSTALLEDVERSION=2.4.180 -} - -up_to_2.4.190() { - echo "Nothing to do for 2.4.190" - INSTALLEDVERSION=2.4.190 -} - -up_to_2.4.200() { - echo "Backing up idstools config..." - suricata_idstools_removal_pre - - touch /opt/so/state/esfleet_logstash_config_pillar - - INSTALLEDVERSION=2.4.200 -} - -up_to_2.4.201() { - echo "Nothing to do for 2.4.201" - - INSTALLEDVERSION=2.4.201 -} - -up_to_2.4.210() { - # Elastic Update for this release, so download Elastic Agent files - determine_elastic_agent_upgrade - create_ca_pillar - # This state is used to deal with the breaking change introduced in 3006.17 - https://docs.saltproject.io/en/3006/topics/releases/3006.17.html - # This is the only way the state is called so we can use concurrent=True - salt-call state.apply salt.master.add_minimum_auth_version --file-root=$UPDATE_DIR/salt --local concurrent=True - INSTALLEDVERSION=2.4.210 -} - add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls