From af53dcda1bc064c7dba0a4fb1927007e7280c2ab Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:32:00 -0400 Subject: [PATCH] Remove references to kafkanode Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- .../assigned_hostgroups.local.map.yaml | 3 +- pillar/kafka/nodes.sls | 2 +- pillar/logstash/nodes.sls | 2 +- pillar/top.sls | 9 +- salt/allowed_states.map.jinja | 14 +-- salt/firewall/containers.map.jinja | 6 +- salt/firewall/defaults.yaml | 92 +------------------ salt/firewall/soc_firewall.yaml | 62 ------------- salt/kafka/enabled.sls | 2 +- salt/logstash/config.sls | 2 +- salt/logstash/defaults.yaml | 4 - salt/logstash/enabled.sls | 2 +- .../config/so/0800_input_kafka.conf.jinja | 4 +- salt/logstash/soc_logstash.yaml | 2 - salt/manager/tools/sbin/so-firewall-minion | 3 - salt/manager/tools/sbin/so-minion | 5 - salt/ssl/init.sls | 5 +- salt/top.sls | 9 -- setup/so-functions | 4 +- setup/so-whiptail | 3 - 20 files changed, 17 insertions(+), 218 deletions(-) diff --git a/files/firewall/assigned_hostgroups.local.map.yaml b/files/firewall/assigned_hostgroups.local.map.yaml index fca293d3a..025b32131 100644 --- a/files/firewall/assigned_hostgroups.local.map.yaml +++ b/files/firewall/assigned_hostgroups.local.map.yaml @@ -19,5 +19,4 @@ role: receiver: standalone: searchnode: - sensor: - kafkanode: \ No newline at end of file + sensor: \ No newline at end of file diff --git a/pillar/kafka/nodes.sls b/pillar/kafka/nodes.sls index b1842834c..6fe64685d 100644 --- a/pillar/kafka/nodes.sls +++ b/pillar/kafka/nodes.sls @@ -1,4 +1,4 @@ -{% set current_kafkanodes = salt.saltutil.runner('mine.get', tgt='G@role:so-kafkanode or G@role:so-manager', fun='network.ip_addrs', tgt_type='compound') %} +{% set current_kafkanodes = salt.saltutil.runner('mine.get', tgt='G@role:so-receiver or G@role:so-manager', fun='network.ip_addrs', tgt_type='compound') %} {% set pillar_kafkanodes = salt['pillar.get']('kafka:nodes', default={}, merge=True) %} {% set existing_ids = [] %} diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index 99fbb857c..a77978821 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -2,7 +2,7 @@ {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet or G@role:so-kafkanode ', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ', fun='network.ip_addrs', tgt_type='compound') | dictsort() %} diff --git a/pillar/top.sls b/pillar/top.sls index 61f4f338f..817767bf7 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -233,15 +233,8 @@ base: - redis.adv_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - - '*_kafkanode': - - logstash.nodes - - logstash.soc_logstash - - logstash.adv_logstash - - minions.{{ grains.id }} - - minions.adv_{{ grains.id }} - - secrets - kafka.nodes + - secrets '*_import': - secrets diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 6fa60c2ea..0fa968658 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -188,16 +188,8 @@ 'telegraf', 'firewall', 'schedule', - 'docker_clean' - ], - 'so-kafkanode': [ - 'kafka', - 'logstash', - 'ssl', - 'telegraf', - 'firewall', - 'schedule', - 'docker_clean' + 'docker_clean', + 'kafka' ], 'so-desktop': [ 'ssl', @@ -214,7 +206,7 @@ {% do allowed_states.append('strelka') %} {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import', 'so-kafkanode'] %} + {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %} {% do allowed_states.append('elasticsearch') %} {% endif %} diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 7efb9abab..02a1b7cac 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -81,11 +81,7 @@ {% set NODE_CONTAINERS = [ 'so-logstash', 'so-redis', -] %} -{% elif GLOBALS.role == 'so-kafkanode' %} -{% set NODE_CONTAINERS = [ - 'so-logstash', - 'so-kafka', + 'so-kafka' ] %} {% elif GLOBALS.role == 'so-idh' %} diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index e51bf5825..0b6d06eda 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -19,7 +19,6 @@ firewall: manager: [] managersearch: [] receiver: [] - kafkanode: [] searchnode: [] self: [] sensor: [] @@ -443,15 +442,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - kafkanode: - portgroups: - - yum - - docker_registry - - influxdb - - elastic_agent_control - - elastic_agent_data - - elastic_agent_update - - sensoroni analyst: portgroups: - nginx @@ -530,9 +520,6 @@ firewall: receiver: portgroups: - salt_manager - kafkanode: - portgroups: - - salt_manager desktop: portgroups: - salt_manager @@ -647,15 +634,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - kafkanode: - portgroups: - - yum - - docker_registry - - influxdb - - elastic_agent_control - - elastic_agent_data - - elastic_agent_update - - sensoroni analyst: portgroups: - nginx @@ -1305,14 +1283,17 @@ firewall: - beats_5044 - beats_5644 - elastic_agent_data + - kafka searchnode: portgroups: - redis - beats_5644 + - kafka managersearch: portgroups: - redis - beats_5644 + - kafka self: portgroups: - redis @@ -1383,73 +1364,6 @@ firewall: portgroups: [] customhostgroup9: portgroups: [] - kafkanode: - chain: - DOCKER-USER: - hostgroups: - searchnode: - portgroups: - - kafka - kafkanode: - portgroups: - - kafka - customhostgroup0: - portgroups: [] - customhostgroup1: - portgroups: [] - customhostgroup2: - portgroups: [] - customhostgroup3: - portgroups: [] - customhostgroup4: - portgroups: [] - customhostgroup5: - portgroups: [] - customhostgroup6: - portgroups: [] - customhostgroup7: - portgroups: [] - customhostgroup8: - portgroups: [] - customhostgroup9: - portgroups: [] - INPUT: - hostgroups: - anywhere: - portgroups: - - ssh - dockernet: - portgroups: - - all - localhost: - portgroups: - - all - self: - portgroups: - - syslog - syslog: - portgroups: - - syslog - customhostgroup0: - portgroups: [] - customhostgroup1: - portgroups: [] - customhostgroup2: - portgroups: [] - customhostgroup3: - portgroups: [] - customhostgroup4: - portgroups: [] - customhostgroup5: - portgroups: [] - customhostgroup6: - portgroups: [] - customhostgroup7: - portgroups: [] - customhostgroup8: - portgroups: [] - customhostgroup9: - portgroups: [] idh: chain: DOCKER-USER: diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 3e4c4355f..28791a705 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -34,7 +34,6 @@ firewall: heavynode: *hostgroupsettings idh: *hostgroupsettings import: *hostgroupsettings - kafkanode: *hostgroupsettings localhost: *ROhostgroupsettingsadv manager: *hostgroupsettings managersearch: *hostgroupsettings @@ -361,8 +360,6 @@ firewall: portgroups: *portgroupsdocker endgame: portgroups: *portgroupsdocker - kafkanode: - portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker desktop: @@ -454,8 +451,6 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - kafkanode: - portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker desktop: @@ -940,63 +935,6 @@ firewall: portgroups: *portgroupshost customhostgroup9: portgroups: *portgroupshost - kafkanode: - chain: - DOCKER-USER: - hostgroups: - searchnode: - portgroups: *portgroupsdocker - kafkanode: - portgroups: *portgroupsdocker - customhostgroup0: - portgroups: *portgroupsdocker - customhostgroup1: - portgroups: *portgroupsdocker - customhostgroup2: - portgroups: *portgroupsdocker - customhostgroup3: - portgroups: *portgroupsdocker - customhostgroup4: - portgroups: *portgroupsdocker - customhostgroup5: - portgroups: *portgroupsdocker - customhostgroup6: - portgroups: *portgroupsdocker - customhostgroup7: - portgroups: *portgroupsdocker - customhostgroup8: - portgroups: *portgroupsdocker - customhostgroup9: - portgroups: *portgroupsdocker - INPUT: - hostgroups: - anywhere: - portgroups: *portgroupshost - dockernet: - portgroups: *portgroupshost - localhost: - portgroups: *portgroupshost - customhostgroup0: - portgroups: *portgroupshost - customhostgroup1: - portgroups: *portgroupshost - customhostgroup2: - portgroups: *portgroupshost - customhostgroup3: - portgroups: *portgroupshost - customhostgroup4: - portgroups: *portgroupshost - customhostgroup5: - portgroups: *portgroupshost - customhostgroup6: - portgroups: *portgroupshost - customhostgroup7: - portgroups: *portgroupshost - customhostgroup8: - portgroups: *portgroupshost - customhostgroup9: - portgroups: *portgroupshost - idh: chain: DOCKER-USER: diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index c2fca70db..ed26297b3 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -7,7 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} -{% set KAFKANODES = salt['pillar.get']('kafka:nodes', {}) %} +{% set KAFKANODES = salt['pillar.get']('kafka:nodes', {}) %} include: - kafka.sostatus diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 402d1ef20..8a59c83b7 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -12,7 +12,7 @@ include: - ssl - {% if GLOBALS.role not in ['so-receiver','so-fleet', 'so-kafkanode'] %} + {% if GLOBALS.role not in ['so-receiver','so-fleet'] %} - elasticsearch {% endif %} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 3ca4570fd..348acb622 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -19,8 +19,6 @@ logstash: - search fleet: - fleet - kafkanode: - - kafkanode defined_pipelines: fleet: - so/0012_input_elastic_agent.conf.jinja @@ -39,8 +37,6 @@ logstash: - so/0900_input_redis.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja - kafkanode: - - so/0899_output_kafka.conf.jinja custom0: [] custom1: [] custom2: [] diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index fcc2ec190..3881ef1f4 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -75,7 +75,7 @@ so-logstash: {% else %} - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro {% endif %} - {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode', 'so-kafkanode' ] %} + {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode' ] %} - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index 1391ce983..85e6729e2 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,11 +1,9 @@ -{% set kafka_brokers = salt['pillar.get']('logstash:nodes:kafkanode', {}) %} +{% set kafka_brokers = salt['pillar.get']('logstash:nodes:receiver', {}) %} {% set kafka_on_mngr = salt ['pillar.get']('logstash:nodes:manager', {}) %} {% set broker_ips = [] %} {% for node, node_data in kafka_brokers.items() %} {% do broker_ips.append(node_data['ip'] + ":9092") %} {% endfor %} - -{# For testing kafka stuff from manager not dedicated kafkanodes #} {% for node, node_data in kafka_on_mngr.items() %} {% do broker_ips.append(node_data['ip'] + ":9092") %} {% endfor %} diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 82fb25bec..3172ff7c5 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -16,7 +16,6 @@ logstash: manager: *assigned_pipelines managersearch: *assigned_pipelines fleet: *assigned_pipelines - kafkanode: *assigned_pipelines defined_pipelines: receiver: &defined_pipelines description: List of pipeline configurations assign to this group. @@ -27,7 +26,6 @@ logstash: fleet: *defined_pipelines manager: *defined_pipelines search: *defined_pipelines - kafkanode: *defined_pipelines custom0: *defined_pipelines custom1: *defined_pipelines custom2: *defined_pipelines diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index 3357e5185..66a0afcea 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,9 +79,6 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - 'KAFKANODE') - so-firewall includehost kafkanode "$IP" --apply - ;; 'DESKTOP') so-firewall includehost desktop "$IP" --apply ;; diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 7b3e6fd3e..34e069ece 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -565,11 +565,6 @@ function createRECEIVER() { add_telegraf_to_minion } -function createKAFKANODE() { - add_logstash_to_minion - # add_telegraf_to_minion -} - function createDESKTOP() { add_desktop_to_minion add_telegraf_to_minion diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 90f9cc64f..f337d62cb 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -664,10 +664,7 @@ elastickeyperms: {%- endif %} -# Roles will need to be modified. Below is just for testing encrypted kafka pipelines -# Remove so-manager. Just inplace for testing -{% if grains['role'] in ['so-manager', 'so-kafkanode', 'so-searchnode'] %} -# Create a cert for Redis encryption +{% if grains['role'] in ['so-manager', 'so-searchnode', 'so-receiver'] %} kafka_key: x509.private_key_managed: - name: /etc/pki/kafka.key diff --git a/salt/top.sls b/salt/top.sls index 289dd462b..ec5e4d738 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -235,16 +235,7 @@ base: - firewall - logstash - redis - - elasticfleet.install_agent_grid - - '*_kafkanode and G@saltversion:{{saltversion}}': - - match: compound - kafka - - logstash - - ssl - - telegraf - - firewall - - docker_clean - elasticfleet.install_agent_grid '*_idh and G@saltversion:{{saltversion}}': diff --git a/setup/so-functions b/setup/so-functions index 070711d63..a669c52fc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1143,7 +1143,7 @@ get_redirect() { get_minion_type() { local minion_type case "$install_type" in - 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'SEARCHNODE' | 'FLEET' | 'IDH' | 'STANDALONE' | 'IMPORT' | 'RECEIVER' | 'DESKTOP' | 'KAFKANODE') + 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'SEARCHNODE' | 'FLEET' | 'IDH' | 'STANDALONE' | 'IMPORT' | 'RECEIVER' | 'DESKTOP') minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]') ;; esac @@ -1505,8 +1505,6 @@ process_installtype() { is_import=true elif [ "$install_type" = 'RECEIVER' ]; then is_receiver=true - elif [ "$install_type" = 'KAFKANODE' ]; then - is_kafka=true elif [ "$install_type" = 'DESKTOP' ]; then is_desktop=true fi diff --git a/setup/so-whiptail b/setup/so-whiptail index a732a9c97..fd9625ec4 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -681,7 +681,6 @@ whiptail_install_type_dist_existing() { "HEAVYNODE" "Sensor + Search Node " \ "IDH" "Intrusion Detection Honeypot Node " \ "RECEIVER" "Receiver Node " \ - "KAFKANODE" "Kafka Broker + Kraft controller" \ 3>&1 1>&2 2>&3 # "HOTNODE" "Add Hot Node (Uses Elastic Clustering)" \ # TODO # "WARMNODE" "Add Warm Node to existing Hot or Search node" \ # TODO @@ -712,8 +711,6 @@ whiptail_install_type_dist_existing() { is_import=true elif [ "$install_type" = 'RECEIVER' ]; then is_receiver=true - elif [ "$install_type" = 'KAFKANODE' ]; then - is_kafka=true elif [ "$install_type" = 'DESKTOP' ]; then is_desktop=true fi