move node_data pillar to logstash:nodes, set extra hosts for filebeat docker

pillar/logstash/nodes.sls

@@ -0,0 +1,29 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    fun='network.ip_addrs',
+    tgt_type='compound') | dictsort() 
+%}
+{%   set hostname = minionid.split('_')[0] %}
+{%   set node_type = minionid.split('_')[1] %}
+{%   if node_type not in node_types.keys() %}
+{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%   else %}
+{%     if hostname not in node_types[node_type] %}
+{%       do node_types[node_type].update({hostname: ip[0]}) %}
+{%     else %}
+{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+logstash:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/node_data/ips.sls

@@ -1,23 +1,33 @@
 {% set node_types = {} %}
+{% set manage_alived = salt.saltutil.runner('manage.alived', show_ip=True) %}
+{% set manager = grains.master %}
+{% set manager_type = manager.split('_')|last %}
 {% for minionid, ip in salt.saltutil.runner('mine.get', tgt='*', fun='network.ip_addrs', tgt_type='glob') | dictsort() %}
 {%   set hostname = minionid.split('_')[0] %}
 {%   set node_type = minionid.split('_')[1] %}
+{%   set is_alive = False %}
+{%   if minionid in manage_alived.keys() %}
+{%     if ip[0] == manage_alived[minionid] %}
+{%       set is_alive = True %}
+{%     endif %}
+{%   endif %}
 {%   if node_type not in node_types.keys() %}
-{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %}
 {%   else %}
 {%     if hostname not in node_types[node_type] %}
-{%       do node_types[node_type].update({hostname: ip[0]}) %}
+{%       do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %}
 {%     else %}
-{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%       do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %}
 {%     endif %}
 {%   endif %}
 {% endfor %}
 
 node_data:
-{% for node_type, values in node_types.items() %}
+{% for node_type, host_values in node_types.items() %}
   {{node_type}}:
-{%   for hostname, ip in values.items() %}
+{%   for hostname, details in host_values.items() %}
     {{hostname}}:
-      ip: {{ip}}
+      ip: {{details.ip}}
+      alive: {{ details.alive }}
 {%   endfor %}
 {% endfor %}

pillar/node_data/test_ping.sls

@@ -1,23 +0,0 @@
-{% set node_types = {} %}
-{% for minionid, test_ping in salt.saltutil.runner('mine.get', tgt='*', fun='test.ping', tgt_type='glob') | dictsort() %}
-{%   set node_type = minionid.split('_')[1] %}
-{%   set hostname = minionid.split('_')[0] %}
-{%   if node_type not in node_types.keys() %}
-{%     do node_types.update({node_type: {hostname: test_ping}}) %}
-{%   else %}
-{%     if hostname not in node_types[node_type] %}
-{%       do node_types[node_type].update({hostname: test_ping}) %}
-{%     else %}
-{%       do node_types[node_type][hostname].update(test_ping) %}
-{%     endif %}
-{%   endif %}
-{% endfor %}
-
-node_data:
-{% for node_type, values in node_types.items() %}
-  {{node_type}}:
-{%   for hostname, test_ping in values.items() %}
-    {{hostname}}:
-      test_ping: {{test_ping}}
-{%   endfor %}
-{% endfor %}

pillar/top.sls

@@ -2,8 +2,7 @@ base:
   '*':
     - patch.needs_restarting
     - logrotate
-    - node_data.ips
-    - node_data.test_ping
+    - logstash.nodes
 
   '*_eval or *_helixsensor or *_heavynode or *_sensor or *_standalone or *_import':
     - match: compound

salt/filebeat/etc/filebeat.yml

@@ -322,9 +322,9 @@ output.logstash:
 
   # The Logstash hosts
   hosts:
-{%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %}
+{%- if grains.role in ['so-sensor', 'so-fleet', 'so-node'] %}
 {%-   set LOADBALANCE = ['false'] %}
-{%-   set node_data = salt['pillar.get']('node_data') %}
+{%-   set node_data = salt['pillar.get']('logstash:nodes') %}
 {%-   for node_type, node_details in node_data.items() | sort %}
 {%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
 {%-       for hostname in node_data[node_type].keys() %}

salt/filebeat/init.sls

@@ -17,12 +17,10 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set LOCALHOSTNAME = salt['grains.get']('host') %}
-{% set MAININT = salt['pillar.get']('host:mainint') %}
-{% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
 {% from 'filebeat/map.jinja' import THIRDPARTY with context %}
 {% from 'filebeat/map.jinja' import SO with context %}
+{% from 'filebeat/map.jinja' import EXTRA_HOSTS with context %}
 {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %}
 
 include:
@@ -111,7 +109,7 @@ so-filebeat:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
     - hostname: so-filebeat
     - user: root
-    - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }}
+    - extra_hosts: {{ EXTRA_HOSTS }}
     - binds:
       - /nsm:/nsm:ro
       - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw

salt/filebeat/map.jinja

@@ -4,3 +4,20 @@
 {% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %}
 {% set SO = SODEFAULTS.securityonion_filebeat %}
 {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#}
+
+{% set role = grains.role %}
+{% set EXTRA_HOSTS = [] %}
+{% set mainint = salt['pillar.get']('host:mainint') %}
+{% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
+{% if role in ['so-sensor', 'so-fleet', 'so-node' ] %}
+  {% set node_data = salt['pillar.get']('logstash:nodes') %}
+  {% for node_type, node_details in node_data.items() | sort %}
+    {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
+      {% for hostname in node_data[node_type].keys() %}
+        {% do EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %}
+      {% endfor %}
+    {% endif %}
+  {% endfor %}
+{% else %}
+  {% do EXTRA_HOSTS.append({grains.host:localhostip}) %}
+{% endif %}

salt/salt/etc/minion.d/mine_functions.conf

@@ -1,4 +1,3 @@
 mine_functions:
-  test.ping: []
   network.ip_addrs:
     - interface: {{ pillar.host.mainint }}