diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index 18c4b39bf..adf43156e 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -2,7 +2,7 @@ {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ', fun='network.ip_addrs', tgt_type='compound') | dictsort() %} diff --git a/pillar/top.sls b/pillar/top.sls index 87cab8c29..86de8709e 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -147,6 +147,8 @@ base: - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.index_templates + - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -155,7 +157,6 @@ base: - adv_global - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - - data.nodestab '*_receiver': - logstash diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 6ccbe6985..ce955e93f 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,18 +1,18 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} +{% from 'logstash/map.jinja' import LOGSTASH_NODES with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} {% if grains.id.split('_') | last in ['manager','managersearch'] %} - {% if REDIS_NODES | length > 1 %} + {% if LOGSTASH_NODES | length > 1 %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% if HIGHLANDER %} {% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'transform']) %} {% endif %} - {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} - {% for SN in REDIS_NODES.keys() %} - {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} + {% for NODE in LOGSTASH_NODES %} + {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'manager' %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index e713e933a..857a3a558 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,7 +15,7 @@ include: {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %} {% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} +{% from 'logstash/map.jinja' import LOGSTASH_NODES %} vm.max_map_count: sysctl.present: @@ -293,9 +293,9 @@ so-elasticsearch: - networks: - sosbridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - - extra_hosts: {{ REDIS_NODES }} + - extra_hosts: {{ LOGSTASH_NODES }} - environment: - {% if REDIS_NODES | length == 1 %} + {% if LOGSTASH_NODES | length == 1 %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 0f16228b2..2af5b2bc6 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -78,6 +78,9 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.docker_registry }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} {% if ISAIRGAP is sameas true %} - {{ portgroups.agrules }} {% endif %} @@ -87,16 +90,31 @@ role: - {{ portgroups.beats_5644 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} searchnodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} + - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} heavynodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} + - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} self: portgroups: - {{ portgroups.syslog}} @@ -133,6 +151,15 @@ role: localhost: portgroups: - {{ portgroups.all }} + sensors: + portgroups: + - {{ portgroups.salt_manager }} + searchnodes: + portgroups: + - {{ portgroups.salt_manager }} + heavynodes: + portgroups: + - {{ portgroups.salt_manager }} managersearch: chain: DOCKER-USER: @@ -146,20 +173,38 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.docker_registry }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} searchnodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} heavynodes: portgroups: - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.yum }} + - {{ portgroups.docker_registry }} + - {{ portgroups.influxdb }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} self: portgroups: - {{ portgroups.syslog}} @@ -196,6 +241,15 @@ role: localhost: portgroups: - {{ portgroups.all }} + sensors: + portgroups: + - {{ portgroups.salt_manager }} + searchnodes: + portgroups: + - {{ portgroups.salt_manager }} + heavynodes: + portgroups: + - {{ portgroups.salt_manager }} standalone: chain: DOCKER-USER: diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index f0363c754..90bce30c6 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -128,6 +128,8 @@ so-grafana: - networks: - sosbridge: - ipv4_address: {{ DOCKER.containers['so-grafana'].ip }} + - extra_hosts: + - {{GLOBALS.influxdb_host}}:{{pillar.node_data[GLOBALS.influxdb_host].ip}} - binds: - /nsm/grafana:/var/lib/grafana:rw - /opt/so/conf/grafana/etc/grafana.ini:/etc/grafana/grafana.ini:ro diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 035e36d86..5f0a7ea79 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,14 +1,20 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% set REDIS_NODES = [] %} -{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %} - {% set node_data = salt['pillar.get']('logstash:nodes') %} - {% for node_type, node_details in node_data.items() | sort %} - {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} - {% for hostname in node_data[node_type].keys() %} - {% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %} - {% endfor %} - {% endif %} - {% endfor %} -{% else %} - {% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %} -{% endif %} +{% set LOGSTASH_NODES = [] %} +{% set node_data = salt['pillar.get']('logstash:nodes') %} + +{% for node_type, node_details in node_data.items() | sort %} +{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %} +{% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} +{% for hostname in node_data[node_type].keys() %} +{% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endif %} +{% else %} +{% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %} +{% endif %} + +{% for hostname in node_data[node_type].keys() %} +{% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endfor %} diff --git a/salt/vars/searchnode.map.jinja b/salt/vars/searchnode.map.jinja new file mode 100644 index 000000000..2efabefed --- /dev/null +++ b/salt/vars/searchnode.map.jinja @@ -0,0 +1,15 @@ +{% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} +{% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} + +{% set ROLE_GLOBALS = {} %} + +{% set STANDALONE_GLOBALS = + [ + ELASTICSEARCH_GLOBALS, + LOGSTASH_GLOBALS + ] +%} + +{% for sg in STANDALONE_GLOBALS %} +{% do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %} +{% endfor %}