From 81fcd68e9b3cb5ebfe0402e41ec0af03f17b4c2d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Jun 2024 16:42:11 -0400 Subject: [PATCH] create and use redis:nodes and elasticsearch:nodes pillars --- pillar/elasticsearch/nodes.sls | 34 +++++++++++++++++++ pillar/logstash/nodes.sls | 9 +++-- pillar/redis/nodes.sls | 34 +++++++++++++++++++ pillar/top.sls | 6 ++++ salt/elasticsearch/config.map.jinja | 23 +++++++++---- salt/elasticsearch/enabled.sls | 8 ++--- .../so-elasticsearch-cluster-settings | 2 +- salt/logstash/map.jinja | 17 ++++++---- .../config/so/0900_input_redis.conf.jinja | 6 ++-- salt/soc/defaults.map.jinja | 2 +- salt/soc/merged.map.jinja | 3 +- 11 files changed, 114 insertions(+), 30 deletions(-) create mode 100644 pillar/elasticsearch/nodes.sls create mode 100644 pillar/redis/nodes.sls diff --git a/pillar/elasticsearch/nodes.sls b/pillar/elasticsearch/nodes.sls new file mode 100644 index 000000000..588b51217 --- /dev/null +++ b/pillar/elasticsearch/nodes.sls @@ -0,0 +1,34 @@ +{% set node_types = {} %} +{% for minionid, ip in salt.saltutil.runner( + 'mine.get', + tgt='elasticsearch:enabled:true', + fun='network.ip_addrs', + tgt_type='pillar') | dictsort() +%} + +# only add a node to the pillar if it returned an ip from the mine +{% if ip | length > 0%} +{% set hostname = minionid.split('_') | first %} +{% set node_type = minionid.split('_') | last %} +{% if node_type not in node_types.keys() %} +{% do node_types.update({node_type: {hostname: ip[0]}}) %} +{% else %} +{% if hostname not in node_types[node_type] %} +{% do node_types[node_type].update({hostname: ip[0]}) %} +{% else %} +{% do node_types[node_type][hostname].update(ip[0]) %} +{% endif %} +{% endif %} +{% endif %} +{% endfor %} + + +elasticsearch: + nodes: +{% for node_type, values in node_types.items() %} + {{node_type}}: +{% for hostname, ip in values.items() %} + {{hostname}}: + ip: {{ip}} +{% endfor %} +{% endfor %} diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index a77978821..4cea7737a 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -1,16 +1,15 @@ {% set node_types = {} %} -{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ', + tgt='logstash:enabled:true', fun='network.ip_addrs', - tgt_type='compound') | dictsort() + tgt_type='pillar') | dictsort() %} # only add a node to the pillar if it returned an ip from the mine {% if ip | length > 0%} -{% set hostname = cached_grains[minionid]['host'] %} -{% set node_type = minionid.split('_')[1] %} +{% set hostname = minionid.split('_') | first %} +{% set node_type = minionid.split('_') | last %} {% if node_type not in node_types.keys() %} {% do node_types.update({node_type: {hostname: ip[0]}}) %} {% else %} diff --git a/pillar/redis/nodes.sls b/pillar/redis/nodes.sls new file mode 100644 index 000000000..618b31715 --- /dev/null +++ b/pillar/redis/nodes.sls @@ -0,0 +1,34 @@ +{% set node_types = {} %} +{% for minionid, ip in salt.saltutil.runner( + 'mine.get', + tgt='redis:enabled:true', + fun='network.ip_addrs', + tgt_type='pillar') | dictsort() +%} + +# only add a node to the pillar if it returned an ip from the mine +{% if ip | length > 0%} +{% set hostname = minionid.split('_') | first %} +{% set node_type = minionid.split('_') | last %} +{% if node_type not in node_types.keys() %} +{% do node_types.update({node_type: {hostname: ip[0]}}) %} +{% else %} +{% if hostname not in node_types[node_type] %} +{% do node_types[node_type].update({hostname: ip[0]}) %} +{% else %} +{% do node_types[node_type][hostname].update(ip[0]) %} +{% endif %} +{% endif %} +{% endif %} +{% endfor %} + + +redis: + nodes: +{% for node_type, values in node_types.items() %} + {{node_type}}: +{% for hostname, ip in values.items() %} + {{hostname}}: + ip: {{ip}} +{% endfor %} +{% endfor %} diff --git a/pillar/top.sls b/pillar/top.sls index b6d6c2e73..d75d3e3a7 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -47,10 +47,12 @@ base: - kibana.adv_kibana - kratos.soc_kratos - kratos.adv_kratos + - redis.nodes - redis.soc_redis - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - elasticfleet.soc_elasticfleet @@ -144,10 +146,12 @@ base: - idstools.adv_idstools - kratos.soc_kratos - kratos.adv_kratos + - redis.nodes - redis.soc_redis - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - elasticfleet.soc_elasticfleet @@ -209,11 +213,13 @@ base: - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash + - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + - redis.nodes - redis.soc_redis - redis.adv_redis - minions.{{ grains.id }} diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 4e57199af..8127df848 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -3,21 +3,30 @@ {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} -{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #} -{% set ES_LOGSTASH_NODES = [] %} -{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #} +{% set ELASTICSEARCH_SEED_HOSTS = [] %} +{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} {% for node_type, node_details in node_data.items() | sort %} -{% if node_type not in ['heavynode', 'fleet'] %} +{% if node_type != 'heavynode' %} {% for hostname in node_data[node_type].keys() %} -{% do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} +{% do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endif %} {% endfor %} +{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #} +{% set ELASTICSEARCH_NODES = [] %} +{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% for node_type, node_details in node_data.items() %} +{% for hostname in node_data[node_type].keys() %} +{% do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endfor %} + {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %} - {% if ES_LOGSTASH_NODES | length > 1 %} + {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} - {% for NODE in ES_LOGSTASH_NODES %} + {% for NODE in ELASTICSEARCH_SEED_HOSTS %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% endif %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 1d35d3505..383fd1cb4 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -7,8 +7,8 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'logstash/map.jinja' import LOGSTASH_NODES %} -{% from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} @@ -27,7 +27,7 @@ so-elasticsearch: - sobridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - extra_hosts: - {% for node in LOGSTASH_NODES %} + {% for node in ELASTICSEARCH_NODES %} {% for hostname, ip in node.items() %} - {{hostname}}:{{ip}} {% endfor %} @@ -38,7 +38,7 @@ so-elasticsearch: {% endfor %} {% endif %} - environment: - {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %} + {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings index 9048b85b1..fb1bfa08b 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings @@ -4,7 +4,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. {%- from 'vars/globals.map.jinja' import GLOBALS %} -{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} . /usr/sbin/so-common diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 4098df21f..5c4b68a76 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -7,23 +7,26 @@ {% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} -{% set REDIS_NODES = [] %} -{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #} +{# used to store the redis nodes that logstash needs to know about to pull from the queue #} +{% set LOGSTASH_REDIS_NODES = [] %} +{# stores all logstash nodes #} {% set LOGSTASH_NODES = [] %} -{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% set logstash_node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% set redis_node_data = salt['pillar.get']('redis:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} -{% for node_type, node_details in node_data.items() | sort %} +{% for node_type, node_details in redis_node_data.items() | sort %} {% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch', 'so-fleet'] %} {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} {% for hostname in node_data[node_type].keys() %} -{% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %} +{% do LOGSTASH_REDIS_NODES.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endif %} -{% else %} -{% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %} {% endif %} +{% endfor %} +{% for node_type, node_details in logstash_node_data.items() | sort %} {% for hostname in node_data[node_type].keys() %} {% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endfor %} + diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index 661bc0b73..ad9fae5f2 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,8 +1,8 @@ -{%- from 'logstash/map.jinja' import REDIS_NODES with context %} +{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} -{%- for index in range(REDIS_NODES|length) %} -{%- for host in REDIS_NODES[index] %} +{%- for index in range(LOGSTASH_REDIS_NODES|length) %} +{%- for host in LOGSTASH_REDIS_NODES[index] %} input { redis { host => '{{ host }}' diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 83cb5637c..b52aa97d8 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -14,7 +14,7 @@ {% endfor %} {# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #} -{% for node_type, minions in salt['pillar.get']('logstash:nodes', {}).items() %} +{% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %} {% if node_type in ['heavynode'] %} {% for m in minions.keys() %} {% do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append('https://' ~ m ~ ':9200') %} diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index c823175cb..a3a28d913 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -5,9 +5,8 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %} -{% from 'logstash/map.jinja' import LOGSTASH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES as DOCKER_EXTRA_HOSTS %} {% from 'manager/map.jinja' import MANAGERMERGED %} -{% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %} {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %} {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}