Merge pull request #9691 from Security-Onion-Solutions/2.4/firewall

2.4/firewall

pillar/logstash/nodes.sls

@@ -2,7 +2,7 @@
 {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
     fun='network.ip_addrs',
     tgt_type='compound') | dictsort()
 %}

pillar/top.sls

@@ -147,6 +147,8 @@ base:
     - logstash.soc_logstash
     - logstash.adv_logstash
     - elasticsearch.index_templates
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
@@ -155,7 +157,6 @@ base:
     - adv_global
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
-    - data.nodestab
 
   '*_receiver':
     - logstash

salt/elasticsearch/config.map.jinja

@@ -1,18 +1,18 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
-{% from 'logstash/map.jinja' import REDIS_NODES with context %}
+{% from 'logstash/map.jinja' import LOGSTASH_NODES with context %}
 
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
 {% if grains.id.split('_') | last in ['manager','managersearch'] %}
-    {% if REDIS_NODES | length > 1 %}
+    {% if LOGSTASH_NODES | length > 1 %}
     {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %}
     {% if HIGHLANDER %}
         {% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'transform']) %}
     {% endif %}
-    {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %}
+    {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
-    {% for SN in REDIS_NODES.keys() %}
+    {% for NODE in LOGSTASH_NODES %}
-      {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN) %}
+      {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
     {% endfor %}
   {% endif %}
   {% if grains.id.split('_') | last == 'manager' %}

salt/elasticsearch/init.sls

@@ -15,7 +15,7 @@ include:
 {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %}
 {% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %}
 {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %}
-{% from 'logstash/map.jinja' import REDIS_NODES with context %}
+{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
 
 vm.max_map_count:
   sysctl.present:
@@ -293,9 +293,9 @@ so-elasticsearch:
     - networks:
       - sosbridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
-    - extra_hosts:  {{ REDIS_NODES }} 
+    - extra_hosts:  {{ LOGSTASH_NODES }}
     - environment:
-      {% if REDIS_NODES | length == 1 %}
+      {% if LOGSTASH_NODES | length == 1 %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true

salt/firewall/assigned_hostgroups.map.yaml

@@ -78,6 +78,9 @@ role:
               - {{ portgroups.influxdb }}
               - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
               {% if ISAIRGAP is sameas true %}
               - {{ portgroups.agrules }}
               {% endif %}
@@ -87,16 +90,31 @@ role:
               - {{ portgroups.beats_5644 }}
               - {{ portgroups.elastic_agent_control }}
               - {{ portgroups.elastic_agent_data }}
+              - {{ portgroups.yum }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.influxdb }}
           searchnodes:
             portgroups:
               - {{ portgroups.redis }}
+              - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
               - {{ portgroups.beats_5644 }}
+              - {{ portgroups.yum }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.influxdb }}
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
           heavynodes:
             portgroups:
               - {{ portgroups.redis }}
+              - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
               - {{ portgroups.beats_5644 }}
+              - {{ portgroups.yum }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.influxdb }}
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
           self:
             portgroups:
               - {{ portgroups.syslog}}
@@ -133,6 +151,15 @@ role:
           localhost:
             portgroups:
               - {{ portgroups.all }}
+          sensors:
+            portgroups:
+              - {{ portgroups.salt_manager }}
+          searchnodes:
+            portgroups:
+              - {{ portgroups.salt_manager }}
+          heavynodes:
+            portgroups:
+              - {{ portgroups.salt_manager }}
   managersearch:
     chain:
       DOCKER-USER:
@@ -146,20 +173,38 @@ role:
               - {{ portgroups.influxdb }}
               - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
           sensors:
             portgroups:
               - {{ portgroups.beats_5044 }}
               - {{ portgroups.beats_5644 }}
               - {{ portgroups.elastic_agent_control }}
               - {{ portgroups.elastic_agent_data }}
+              - {{ portgroups.yum }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.influxdb }}
           searchnodes:
             portgroups:
               - {{ portgroups.redis }}
+              - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.yum }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.influxdb }}
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
           heavynodes:
             portgroups:
               - {{ portgroups.redis }}
+              - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.yum }}
+              - {{ portgroups.docker_registry }}
+              - {{ portgroups.influxdb }}
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
           self:
             portgroups:
               - {{ portgroups.syslog}}
@@ -196,6 +241,15 @@ role:
           localhost:
             portgroups:
               - {{ portgroups.all }}
+          sensors:
+            portgroups:
+              - {{ portgroups.salt_manager }}
+          searchnodes:
+            portgroups:
+              - {{ portgroups.salt_manager }}
+          heavynodes:
+            portgroups:
+              - {{ portgroups.salt_manager }}
   standalone:
     chain:
       DOCKER-USER:

salt/grafana/init.sls

@@ -128,6 +128,8 @@ so-grafana:
     - networks:
       - sosbridge:
         - ipv4_address: {{ DOCKER.containers['so-grafana'].ip }}
+    - extra_hosts:
+      - {{GLOBALS.influxdb_host}}:{{pillar.node_data[GLOBALS.influxdb_host].ip}}
     - binds:
       - /nsm/grafana:/var/lib/grafana:rw
       - /opt/so/conf/grafana/etc/grafana.ini:/etc/grafana/grafana.ini:ro

salt/logstash/map.jinja

@@ -1,14 +1,20 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% set REDIS_NODES = [] %}
-{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %}
+{% set LOGSTASH_NODES = [] %}
-  {% set node_data = salt['pillar.get']('logstash:nodes') %}
+{% set node_data = salt['pillar.get']('logstash:nodes') %}
-  {% for node_type, node_details in node_data.items() | sort %}
+
-    {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
+{% for node_type, node_details in node_data.items() | sort %}
-      {% for hostname in node_data[node_type].keys() %}
+{%   if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %}
-        {% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
+{%     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
-      {% endfor %}
+{%       for hostname in node_data[node_type].keys() %}
-    {% endif %}
+{%         do REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
-  {% endfor %}
+{%       endfor %}
-{% else %}
+{%     endif %}
-  {% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %}
+{%   else %}
-{% endif %}
+{%     do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %}
+{%   endif %}
+
+{%   for hostname in node_data[node_type].keys() %}
+{%     do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%   endfor %}
+{% endfor %}

salt/vars/searchnode.map.jinja

@@ -0,0 +1,15 @@
+{% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
+{% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %}
+
+{% set ROLE_GLOBALS = {} %}
+
+{% set STANDALONE_GLOBALS =
+   [
+     ELASTICSEARCH_GLOBALS,
+     LOGSTASH_GLOBALS
+   ]
+%}
+
+{% for sg in STANDALONE_GLOBALS %}
+{%   do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %}
+{% endfor %}