From f2d3298f14c4d71d973a7e82c2f0e76bb307ab9c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 26 Jan 2023 13:54:52 -0500 Subject: [PATCH 01/10] allow nodes to connect to salt for manager and managersearch --- salt/firewall/assigned_hostgroups.map.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 6e249ec93..2950dc8be 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -133,6 +133,15 @@ role: localhost: portgroups: - {{ portgroups.all }} + sensors: + portgroups: + - {{ portgroups.salt_manager }} + searchnodes: + portgroups: + - {{ portgroups.salt_manager }} + heavynodes: + portgroups: + - {{ portgroups.salt_manager }} managersearch: chain: DOCKER-USER: @@ -196,6 +205,15 @@ role: localhost: portgroups: - {{ portgroups.all }} + sensors: + portgroups: + - {{ portgroups.salt_manager }} + searchnodes: + portgroups: + - {{ portgroups.salt_manager }} + heavynodes: + portgroups: + - {{ portgroups.salt_manager }} standalone: chain: DOCKER-USER: From 75d73e4620aeac460e98f0f48317b4e8b6ef2707 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 26 Jan 2023 15:35:22 -0500 Subject: [PATCH 02/10] add yum portgroups for amnager --- salt/firewall/assigned_hostgroups.map.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index fc0a629e7..3fd3fd1f3 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -87,16 +87,19 @@ role: - {{ portgroups.beats_5644 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.yum }} searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} + - {{ portgroups.yum }} heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} + - {{ portgroups.yum }} self: portgroups: - {{ portgroups.syslog}} @@ -161,14 +164,17 @@ role: - {{ portgroups.beats_5644 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.yum }} searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.yum }} heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.yum }} self: portgroups: - {{ portgroups.syslog}} From d5b5a36f28521a12e08e34097bd2cbd54849339b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 26 Jan 2023 16:17:33 -0500 Subject: [PATCH 03/10] remove data.nodestab from searchnodes pillar --- pillar/top.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/pillar/top.sls b/pillar/top.sls index 87cab8c29..d383f963a 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -155,7 +155,6 @@ base: - adv_global - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - - data.nodestab '*_receiver': - logstash From 8dc7a9da9e6dfa5c7bb11233c246bdc588386a6b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 31 Jan 2023 10:52:35 -0500 Subject: [PATCH 04/10] add searchnode global vars --- salt/vars/searchnode.map.jinja | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 salt/vars/searchnode.map.jinja diff --git a/salt/vars/searchnode.map.jinja b/salt/vars/searchnode.map.jinja new file mode 100644 index 000000000..2efabefed --- /dev/null +++ b/salt/vars/searchnode.map.jinja @@ -0,0 +1,15 @@ +{% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} +{% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} + +{% set ROLE_GLOBALS = {} %} + +{% set STANDALONE_GLOBALS = + [ + ELASTICSEARCH_GLOBALS, + LOGSTASH_GLOBALS + ] +%} + +{% for sg in STANDALONE_GLOBALS %} +{% do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %} +{% endfor %} From e09a86dc3067f9299451d4df4c62b68b3e4f8eb7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 31 Jan 2023 10:54:40 -0500 Subject: [PATCH 05/10] 2.4 searchnode es config --- pillar/logstash/nodes.sls | 2 +- pillar/top.sls | 2 ++ salt/elasticsearch/config.map.jinja | 10 +++++----- salt/elasticsearch/init.sls | 6 +++--- salt/logstash/map.jinja | 30 +++++++++++++++++------------ 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index 18c4b39bf..adf43156e 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -2,7 +2,7 @@ {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ', fun='network.ip_addrs', tgt_type='compound') | dictsort() %} diff --git a/pillar/top.sls b/pillar/top.sls index d383f963a..86de8709e 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -147,6 +147,8 @@ base: - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.index_templates + - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 6ccbe6985..ce955e93f 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,18 +1,18 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} +{% from 'logstash/map.jinja' import LOGSTASH_NODES with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} {% if grains.id.split('_') | last in ['manager','managersearch'] %} - {% if REDIS_NODES | length > 1 %} + {% if LOGSTASH_NODES | length > 1 %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% if HIGHLANDER %} {% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'transform']) %} {% endif %} - {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} - {% for SN in REDIS_NODES.keys() %} - {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} + {% for NODE in LOGSTASH_NODES %} + {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'manager' %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index e713e933a..857a3a558 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,7 +15,7 @@ include: {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %} {% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} +{% from 'logstash/map.jinja' import LOGSTASH_NODES %} vm.max_map_count: sysctl.present: @@ -293,9 +293,9 @@ so-elasticsearch: - networks: - sosbridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - - extra_hosts: {{ REDIS_NODES }} + - extra_hosts: {{ LOGSTASH_NODES }} - environment: - {% if REDIS_NODES | length == 1 %} + {% if LOGSTASH_NODES | length == 1 %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 035e36d86..5f0a7ea79 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,14 +1,20 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% set REDIS_NODES = [] %} -{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %} - {% set node_data = salt['pillar.get']('logstash:nodes') %} - {% for node_type, node_details in node_data.items() | sort %} - {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} - {% for hostname in node_data[node_type].keys() %} - {% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %} - {% endfor %} - {% endif %} - {% endfor %} -{% else %} - {% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %} -{% endif %} +{% set LOGSTASH_NODES = [] %} +{% set node_data = salt['pillar.get']('logstash:nodes') %} + +{% for node_type, node_details in node_data.items() | sort %} +{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %} +{% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} +{% for hostname in node_data[node_type].keys() %} +{% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endif %} +{% else %} +{% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %} +{% endif %} + +{% for hostname in node_data[node_type].keys() %} +{% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endfor %} From 0ba193c7a4d8fe0f87f258a999f6eff88212dcf6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 31 Jan 2023 10:55:14 -0500 Subject: [PATCH 06/10] allow docker_registry fw --- salt/firewall/assigned_hostgroups.map.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 3fd3fd1f3..8851e3051 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -78,6 +78,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.docker_registry }} {% if ISAIRGAP is sameas true %} - {{ portgroups.agrules }} {% endif %} @@ -88,18 +89,21 @@ role: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} self: portgroups: - {{ portgroups.syslog}} @@ -158,6 +162,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.docker_registry }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -165,16 +170,19 @@ role: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} self: portgroups: - {{ portgroups.syslog}} From aa411e2682db4af2c101d855197e1d4741d8a9cd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 31 Jan 2023 12:42:46 -0500 Subject: [PATCH 07/10] allow influxdb on manager and managersearch --- salt/firewall/assigned_hostgroups.map.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 8851e3051..e31b7dbba 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -90,6 +90,7 @@ role: - {{ portgroups.elastic_agent_data }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} searchnodes: portgroups: - {{ portgroups.redis }} @@ -97,6 +98,7 @@ role: - {{ portgroups.beats_5644 }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} heavynodes: portgroups: - {{ portgroups.redis }} @@ -104,6 +106,7 @@ role: - {{ portgroups.beats_5644 }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} self: portgroups: - {{ portgroups.syslog}} @@ -171,18 +174,21 @@ role: - {{ portgroups.elastic_agent_data }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} self: portgroups: - {{ portgroups.syslog}} From 16e1e297a057151162438be328c760f45f5a9bad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 31 Jan 2023 13:32:33 -0500 Subject: [PATCH 08/10] allow elasticsearch_rest --- salt/firewall/assigned_hostgroups.map.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index e31b7dbba..f0c0a7223 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -94,6 +94,7 @@ role: searchnodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} - {{ portgroups.yum }} @@ -102,6 +103,7 @@ role: heavynodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} - {{ portgroups.yum }} @@ -178,6 +180,7 @@ role: searchnodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} @@ -185,6 +188,7 @@ role: heavynodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} From 6f1438148f84e9e403850f63a7efb4fb703ebfd4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 31 Jan 2023 16:54:46 -0500 Subject: [PATCH 09/10] allow elastic agent access --- salt/firewall/assigned_hostgroups.map.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index f0c0a7223..2af5b2bc6 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -79,6 +79,8 @@ role: - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.docker_registry }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} {% if ISAIRGAP is sameas true %} - {{ portgroups.agrules }} {% endif %} @@ -100,6 +102,8 @@ role: - {{ portgroups.yum }} - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} heavynodes: portgroups: - {{ portgroups.redis }} @@ -109,6 +113,8 @@ role: - {{ portgroups.yum }} - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} self: portgroups: - {{ portgroups.syslog}} @@ -168,6 +174,8 @@ role: - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.docker_registry }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -185,6 +193,8 @@ role: - {{ portgroups.yum }} - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} heavynodes: portgroups: - {{ portgroups.redis }} @@ -193,6 +203,8 @@ role: - {{ portgroups.yum }} - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} self: portgroups: - {{ portgroups.syslog}} From 6ec086e24af0e460077e98bd1ccf3d9dc8391f09 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 31 Jan 2023 17:10:11 -0500 Subject: [PATCH 10/10] add influxdb as extra_hosts for grafana container --- salt/grafana/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index f0363c754..90bce30c6 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -128,6 +128,8 @@ so-grafana: - networks: - sosbridge: - ipv4_address: {{ DOCKER.containers['so-grafana'].ip }} + - extra_hosts: + - {{GLOBALS.influxdb_host}}:{{pillar.node_data[GLOBALS.influxdb_host].ip}} - binds: - /nsm/grafana:/var/lib/grafana:rw - /opt/so/conf/grafana/etc/grafana.ini:/etc/grafana/grafana.ini:ro