diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 02624d813..0748557fd 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -79,7 +79,7 @@ printf "\n\n" printf "\nCreate Kafka Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} -salt-call state.apply kafka.elasticfleet queue=True +/usr/sbin/so-kafka-fleet-output-policy {% endif %} # Add Manager Hostname & URL Base to Fleet Host URLs diff --git a/salt/kafka/tools/sbin_jinja/so-kafka-fleet-output-policy b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy similarity index 69% rename from salt/kafka/tools/sbin_jinja/so-kafka-fleet-output-policy rename to salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy index 7a1f4e1c5..c3cfd34db 100644 --- a/salt/kafka/tools/sbin_jinja/so-kafka-fleet-output-policy +++ b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy @@ -4,7 +4,21 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -output=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id) +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + +# Check to make sure that Kibana API is up & ready +RETURN_CODE=0 +wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" +RETURN_CODE=$? + +if [[ "$RETURN_CODE" != "0" ]]; then + printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..." + exit 1 +fi + +output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id) if ! echo "$output" | grep -q "so-manager_kafka"; then KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt) @@ -19,5 +33,5 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \ '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }' ) - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 2&1> /dev/null + curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" fi \ No newline at end of file diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 1c3d8c26b..d7d315a7c 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -39,7 +39,7 @@ kafka_sbin_jinja_tools: - defaults: GLOBALS: {{ GLOBALS }} -kakfa_log_dir: +kafka_log_dir: file.directory: - name: /opt/so/log/kafka - user: 960 diff --git a/salt/kafka/elasticfleet.sls b/salt/kafka/elasticfleet.sls deleted file mode 100644 index ae8899821..000000000 --- a/salt/kafka/elasticfleet.sls +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'vars/globals.map.jinja' import GLOBALS %} -include: - - elasticfleet.enabled - -{# Create Kafka output policy if it doesn't exist #} -update_kafka_output_policy_script: - file.managed: - - name: /usr/sbin/so-kafka-fleet-output-policy - - source: salt://kafka/tools/sbin_jinja/so-kafka-fleet-output-policy - - user: root - - mode: 755 - - template: jinja - - defaults: - GLOBALS: {{ GLOBALS }} - -create_kafka_output_policy: - cmd.run: - - name: 'so-kafka-fleet-output-policy > /dev/null 2>&1' - - show_changes: false \ No newline at end of file diff --git a/salt/kafka/init.sls b/salt/kafka/init.sls index 49707033e..0a9b3305f 100644 --- a/salt/kafka/init.sls +++ b/salt/kafka/init.sls @@ -16,7 +16,6 @@ include: {# Run kafka/nodes.sls before Kafka is enabled, so kafka nodes pillar is setup #} {% if grains.role in ['so-manager','so-managersearch', 'so-standalone'] %} - kafka.nodes - - kafka.elasticfleet {% endif %} {% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %} - kafka.enabled diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index 33a4d39e4..dfb246210 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,8 +1,8 @@ {%- set kafka_password = salt['pillar.get']('kafka:password') %} -{%- set kafka_brokers = salt['pillar.get']('kafka:nodes', default=None) %} +{%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} {%- set brokers = [] %} -{%- if kafka_brokers != none %} +{%- if kafka_brokers %} {%- for key, values in kafka_brokers.items() %} {%- if 'broker' in values['role'] %} {%- do brokers.append(key ~ ':9092') %} diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 226c81f9d..a2828a1ea 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -477,7 +477,8 @@ post_to_2.4.70() { } post_to_2.4.80() { - echo "Nothing to apply" + echo -e "\nUpdating Elastic Fleet output policy\n" + so-kafka-fleet-output-policy POSTVERSION=2.4.80 } @@ -636,15 +637,15 @@ up_to_2.4.80() { # Global pipeline changes to REDIS or KAFKA echo "Removing global.pipeline pillar configuration" sed -i '/pipeline:/d' /opt/so/saltstack/local/pillar/global/soc_global.sls - # Kafka pillars + # Kafka pillars mkdir -p /opt/so/saltstack/local/pillar/kafka touch /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls touch /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls echo 'kafka: ' > /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka_cluster_id=$(get_random_value 22) - echo ' cluster_id: '$kafka_cluster_id >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + echo ' cluster_id: '$kafka_cluster_id >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafkapass=$(get_random_value) - echo ' password: '$kafkapass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + echo ' password: '$kafkapass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls INSTALLEDVERSION=2.4.80 } diff --git a/setup/so-functions b/setup/so-functions index 41b450d67..b1469b7eb 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1180,7 +1180,7 @@ kibana_pillar() { kafka_pillar() { KAFKACLUSTERID=$(get_random_value 22) KAFKAPASS=$(get_random_value) - logCmd "mkdir -p $local_salt_dir/pillar/kakfa" + logCmd "mkdir -p $local_salt_dir/pillar/kafka" logCmd "touch $adv_kafka_pillar_file" logCmd "touch $kafka_pillar_file" printf '%s\n'\ diff --git a/setup/so-variables b/setup/so-variables index 4a2f29c58..ecc29b554 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -178,7 +178,7 @@ export redis_pillar_file adv_redis_pillar_file="$local_salt_dir/pillar/redis/adv_redis.sls" export adv_redis_pillar_file -kafka_pillar_file="local_salt_dir/pillar/kafka/soc_kafka.sls" +kafka_pillar_file="$local_salt_dir/pillar/kafka/soc_kafka.sls" export kafka_pillar_file adv_kafka_pillar_file="$local_salt_dir/pillar/kafka/adv_kafka.sls"