Merge pull request #13209 from Security-Onion-Solutions/kfix

Fix errors on new installs

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -79,7 +79,7 @@ printf "\n\n"
 
 printf "\nCreate Kafka Output Config if node is not an Import or Eval install\n"
 {% if grains.role not in ['so-import', 'so-eval'] %}
-salt-call state.apply kafka.elasticfleet queue=True
+/usr/sbin/so-kafka-fleet-output-policy
 {% endif %}
 
 # Add Manager Hostname & URL Base to Fleet Host URLs

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -4,7 +4,21 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-output=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+# Check to make sure that Kibana API is up & ready
+RETURN_CODE=0
+wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
+RETURN_CODE=$?
+
+if [[ "$RETURN_CODE" != "0" ]]; then
+    printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..."
+    exit 1
+fi
+
+output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
 
 if ! echo "$output" | grep -q "so-manager_kafka"; then
   KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
@@ -19,5 +33,5 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then
                 --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
                     '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
                 )
-  curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 2&1> /dev/null
+  curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 fi

salt/kafka/config.sls

@@ -39,7 +39,7 @@ kafka_sbin_jinja_tools:
     - defaults:
         GLOBALS: {{ GLOBALS }}
 
-kakfa_log_dir:
+kafka_log_dir:
   file.directory:
     - name: /opt/so/log/kafka
     - user: 960

salt/kafka/elasticfleet.sls

@@ -1,24 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-include:
-  - elasticfleet.enabled
-
-{# Create Kafka output policy if it doesn't exist #}
-update_kafka_output_policy_script:
-  file.managed:
-    - name: /usr/sbin/so-kafka-fleet-output-policy
-    - source: salt://kafka/tools/sbin_jinja/so-kafka-fleet-output-policy
-    - user: root
-    - mode: 755
-    - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
-
-create_kafka_output_policy:
-  cmd.run:
-    - name: 'so-kafka-fleet-output-policy > /dev/null 2>&1'
-    - show_changes: false

salt/kafka/init.sls

@@ -16,7 +16,6 @@ include:
 {# Run kafka/nodes.sls before Kafka is enabled, so kafka nodes pillar is setup #}
 {% if grains.role in ['so-manager','so-managersearch', 'so-standalone'] %}
   - kafka.nodes
-  - kafka.elasticfleet
 {% endif %}
 {% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %}
   - kafka.enabled

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,8 +1,8 @@
 {%- set kafka_password = salt['pillar.get']('kafka:password') %}
-{%- set kafka_brokers = salt['pillar.get']('kafka:nodes', default=None) %}
+{%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
 {%- set brokers = [] %}
 
-{%- if kafka_brokers != none %}
+{%- if kafka_brokers %}
 {%-   for key, values in kafka_brokers.items() %}
 {%-     if 'broker' in values['role'] %}
 {%-       do brokers.append(key ~ ':9092') %}

salt/manager/tools/sbin/soup

@@ -477,7 +477,8 @@ post_to_2.4.70() {
 }
 
 post_to_2.4.80() {
-  echo "Nothing to apply"
+  echo -e "\nUpdating Elastic Fleet output policy\n"
+  so-kafka-fleet-output-policy
   POSTVERSION=2.4.80
 }
 
@@ -636,15 +637,15 @@ up_to_2.4.80() {
   # Global pipeline changes to REDIS or KAFKA
   echo "Removing global.pipeline pillar configuration"
   sed -i '/pipeline:/d' /opt/so/saltstack/local/pillar/global/soc_global.sls
-    # Kafka pillars
+  # Kafka pillars
   mkdir -p /opt/so/saltstack/local/pillar/kafka
   touch /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
   touch /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls
   echo 'kafka: ' > /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
   kafka_cluster_id=$(get_random_value 22)
-  echo '    cluster_id: '$kafka_cluster_id >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  echo '  cluster_id: '$kafka_cluster_id >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
   kafkapass=$(get_random_value)
-  echo '    password: '$kafkapass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  echo '  password: '$kafkapass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
 
   INSTALLEDVERSION=2.4.80
 }

setup/so-functions

@@ -1180,7 +1180,7 @@ kibana_pillar() {
 kafka_pillar() {
 	KAFKACLUSTERID=$(get_random_value 22)
 	KAFKAPASS=$(get_random_value)
-	logCmd "mkdir -p $local_salt_dir/pillar/kakfa"
+	logCmd "mkdir -p $local_salt_dir/pillar/kafka"
 	logCmd "touch $adv_kafka_pillar_file"
 	logCmd "touch $kafka_pillar_file"
 	printf '%s\n'\

setup/so-variables

@@ -178,7 +178,7 @@ export redis_pillar_file
 adv_redis_pillar_file="$local_salt_dir/pillar/redis/adv_redis.sls"
 export adv_redis_pillar_file
 
-kafka_pillar_file="local_salt_dir/pillar/kafka/soc_kafka.sls"
+kafka_pillar_file="$local_salt_dir/pillar/kafka/soc_kafka.sls"
 export kafka_pillar_file
 
 adv_kafka_pillar_file="$local_salt_dir/pillar/kafka/adv_kafka.sls"