merge dev

salt/manager/init.sls

@@ -73,6 +73,15 @@ manager_sbin:
     - exclude_pat:
       - "*_test.py"
 
+manager_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin/
+    - source: salt://manager/tools/sbin_jinja/
+    - user: socore
+    - group: socore
+    - file_mode: 755
+    - template: jinja
+
 so-repo-file:
   file.managed:
     - name: /opt/so/conf/reposync/repodownload.conf

salt/manager/soc_manager.yaml

@@ -29,6 +29,7 @@ manager:
     global: True
     advanced: True
     multiline: True
+    forcedType: string
     helpLink: proxy.html
   insecureSkipVerify:
     description: Disable TLS verification for outgoing requests. This will make your installation less secure to MITM attacks. Recommended only for debugging purposes.

salt/manager/tools/sbin/so-minion

@@ -603,6 +603,10 @@ function updateMineAndApplyStates() {
 	if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then
 		salt-run state.orch orch.container_download pillar="{'setup': {'newnode': $MINION_ID }}" > /dev/null 2>&1 &
 	fi
+	if [[ "$NODETYPE" == "RECEIVER" ]]; then
+		# Setup nodeid for Kafka
+		salt-call state.apply kafka.nodes queue=True
+	fi
 	# $MINIONID is the minion id of the manager and $MINION_ID is the target node or the node being configured
 	salt-run state.orch orch.deploy_newnode pillar="{'setup': {'manager': $MINIONID, 'newnode': $MINION_ID }}" > /dev/null 2>&1 &
 }

salt/manager/tools/sbin/soup

@@ -492,7 +492,8 @@ post_to_2.4.70() {
 }
 
 post_to_2.4.80() {
-  disable_logstash_heavynodes
+  echo -e "\nChecking if update to Elastic Fleet output policy is required\n"
+  so-kafka-fleet-output-policy
   POSTVERSION=2.4.80
 }
 
@@ -640,11 +641,27 @@ up_to_2.4.70() {
   suricata_idstools_migration
   toggle_telemetry
   add_detection_test_pillars
+
   INSTALLEDVERSION=2.4.70
 }
 
 up_to_2.4.80() {
   phases_pillar_2_4_80
+  # Kafka configuration changes
+
+  # Global pipeline changes to REDIS or KAFKA
+  echo "Removing global.pipeline pillar configuration"
+  sed -i '/pipeline:/d' /opt/so/saltstack/local/pillar/global/soc_global.sls
+  # Kafka pillars
+  mkdir -p /opt/so/saltstack/local/pillar/kafka
+  touch /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  touch /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls
+  echo 'kafka: ' > /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  kafka_cluster_id=$(get_random_value 22)
+  echo '  cluster_id: '$kafka_cluster_id >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  kafkapass=$(get_random_value)
+  echo '  password: '$kafkapass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+
   INSTALLEDVERSION=2.4.80
 }