casing

Serve /kernelrepo through nginx so minions can reach the kernel repo

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,7 +63,8 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS |\
+    sed 's/First packet/Earliest packet/g' | sed 's/Last packet/Latest packet/g'
 }
 
 function pcapfix() {

salt/elasticfleet/config.sls

@@ -173,7 +173,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/manager.sls

@@ -67,8 +67,6 @@ so-elastic-fleet-package-upgrade:
         interval: 30
     - require:
       - http: wait_for_so-kibana
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -9,13 +9,11 @@
 RETURN_CODE=0
 
 if [ ! -f /opt/so/state/eaintegrations.txt ]; then
-  # First, check for any package upgrades
-  /usr/sbin/so-elastic-fleet-package-upgrade
 
-  # Second, update Fleet Server policies
+  # update Fleet Server policies
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
-  # Third, configure Elastic Defend Integration seperately
+  # configure Elastic Defend Integration separately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
   # Each group fetches its agent policy once and dispatches create/update writes concurrently.
@@ -32,9 +30,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \
     /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1
 
-  # Fleet Server - Optional integrations (one agent policy per FleetServer_* directory)
+  # Fleet Server - Optional integrations (adds integration configuration to a given FleetServer_ policy)
   for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do
     [ -d "$FLEET_DIR" ] || continue
+    INTEGRATIONS=("${FLEET_DIR%/}"/*.json)
+    [ -e "${INTEGRATIONS[0]}" ] || continue
+
     FLEET_POLICY=$(basename "$FLEET_DIR")
     elastic_fleet_load_integrations_dir "$FLEET_POLICY" \
       "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -12,17 +12,22 @@ PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
-echo "Upgrading {{ PACKAGE }} package..."
-if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
-    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+
+    if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+        echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
+    else
+        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
+        if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        fi
     fi
 else
+    echo "ERROR: Failed to get version information for integration {{ PACKAGE }}"
     PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
     PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
-echo
 {%- endfor %}
 
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
@@ -35,6 +40,3 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then
 else
     echo "Successfully upgraded all packages."
 fi
-
-echo
-/usr/sbin/so-elasticsearch-templates-load

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -181,6 +181,9 @@ if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy No
     exit 1
 fi
 
+# Check for package upgrades
+so-elastic-fleet-package-upgrade
+
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load
 

salt/elasticsearch/files/ingest/zeek.ssl

@@ -5,6 +5,7 @@
     { "remove":   { "field": ["host"],     "ignore_failure": true } },
     { "json":             { "field": "message",                 "target_field": "message2",             "ignore_failure": true  } },
     { "rename":         { "field": "message2.version",          "target_field": "ssl.version",          "ignore_missing": true  } },
+    { "set":      { "description": "Set transport for the community_id processor", "if": "ctx.ssl?.version == null || !ctx.ssl.version.startsWith('DTLS')", "field": "network.transport", "value": "tcp", "ignore_failure": true } },
     { "rename":         { "field": "message2.cipher",           "target_field": "ssl.cipher",           "ignore_missing": true  } },
     { "rename":         { "field": "message2.curve",            "target_field": "ssl.curve",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.server_name",      "target_field": "ssl.server_name",              "ignore_missing": true  } },

salt/manager/tools/sbin/soup

@@ -850,6 +850,28 @@ kibana_backport_streams_index_template() {
 
 }
 
+# Runs kafka-features.sh upgrade --release-version $1
+#   Upgrades Kafka KRaft cluster metadata
+update_kafka_metadata() {
+    metadata_version="$1"
+    global_pillar="/opt/so/saltstack/local/pillar/global/soc_global.sls"
+    if PIPELINE=$(so-yaml.py get -r "$global_pillar" global.pipeline 2> /dev/null) && [[ "$PIPELINE" == "KAFKA" ]]; then
+        kafka_nodes_raw=$(salt-call pillar.get kafka:nodes --out=json)
+        if kafka_nodes=$(jq -er '.local | select(type == "object" and length > 0)' <<< "$kafka_nodes_raw"); then
+            bootstrap_servers=$(jq -r '[to_entries[] | select(.value.role | contains("broker")) | "\(.value.ip):9092"] | join(",")' <<< "$kafka_nodes")
+            echo "Upgrading Kafka KRaft cluster version"
+            so-kafka-cli kafka-features.sh --bootstrap-server "$bootstrap_servers" --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version "$metadata_version" 2>/dev/null || true
+
+            return 0
+        else
+            FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically perform Kafka KRaft cluster metadata update. This step can be performed manually using the following command (replacing \$BROKER_IP with the ip of atleast 1 available Kafka broker):")
+            FINAL_MESSAGE_QUEUE+=("    - so-kafka-cli kafka-features.sh --bootstrap-server \$BROKER_IP:9092 --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version $metadata_version")
+        fi
+    else
+        echo "Nothing to do!"
+    fi
+}
+
 up_to_3.2.0() {
   fix_logstash_0013_lumberjack_pipeline_name
 
@@ -867,6 +889,8 @@ post_to_3.2.0() {
 
   kibana_backport_streams_index_template
 
+  update_kafka_metadata "4.3"
+
   POSTVERSION=3.2.0
 }
 

setup/so-setup

@@ -9,14 +9,17 @@
 # Make sure you are root before doing anything
 uid="$(id -u)"
 if [ "$uid" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	fail_setup
+	echo "This script must be run using sudo!" >&2
+	exit 1
 fi
 
 # Save the original argument array since we modify it
 original_args=("$@")
 
-cd "$(dirname "$0")" || fail_setup
+cd "$(dirname "$0")" || {
+	echo "Unable to change to setup directory" >&2
+	exit 1
+}
 
 echo "Getting started..."