diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls new file mode 100644 index 000000000..3658065cb --- /dev/null +++ b/pillar/logstash/nodes.sls @@ -0,0 +1,29 @@ +{% set node_types = {} %} +{% for minionid, ip in salt.saltutil.runner( + 'mine.get', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ', + fun='network.ip_addrs', + tgt_type='compound') | dictsort() +%} +{% set hostname = minionid.split('_')[0] %} +{% set node_type = minionid.split('_')[1] %} +{% if node_type not in node_types.keys() %} +{% do node_types.update({node_type: {hostname: ip[0]}}) %} +{% else %} +{% if hostname not in node_types[node_type] %} +{% do node_types[node_type].update({hostname: ip[0]}) %} +{% else %} +{% do node_types[node_type][hostname].update(ip[0]) %} +{% endif %} +{% endif %} +{% endfor %} + +logstash: + nodes: +{% for node_type, values in node_types.items() %} + {{node_type}}: +{% for hostname, ip in values.items() %} + {{hostname}}: + ip: {{ip}} +{% endfor %} +{% endfor %} diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls index 40551eafc..233038e66 100644 --- a/pillar/node_data/ips.sls +++ b/pillar/node_data/ips.sls @@ -1,23 +1,33 @@ {% set node_types = {} %} +{% set manage_alived = salt.saltutil.runner('manage.alived', show_ip=True) %} +{% set manager = grains.master %} +{% set manager_type = manager.split('_')|last %} {% for minionid, ip in salt.saltutil.runner('mine.get', tgt='*', fun='network.ip_addrs', tgt_type='glob') | dictsort() %} {% set hostname = minionid.split('_')[0] %} {% set node_type = minionid.split('_')[1] %} +{% set is_alive = False %} +{% if minionid in manage_alived.keys() %} +{% if ip[0] == manage_alived[minionid] %} +{% set is_alive = True %} +{% endif %} +{% endif %} {% if node_type not in node_types.keys() %} -{% do node_types.update({node_type: {hostname: ip[0]}}) %} +{% do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %} {% else %} {% if hostname not in node_types[node_type] %} -{% do node_types[node_type].update({hostname: ip[0]}) %} +{% do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %} {% else %} -{% do node_types[node_type][hostname].update(ip[0]) %} +{% do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %} {% endif %} {% endif %} {% endfor %} node_data: -{% for node_type, values in node_types.items() %} +{% for node_type, host_values in node_types.items() %} {{node_type}}: -{% for hostname, ip in values.items() %} +{% for hostname, details in host_values.items() %} {{hostname}}: - ip: {{ip}} + ip: {{details.ip}} + alive: {{ details.alive }} {% endfor %} {% endfor %} diff --git a/pillar/node_data/test_ping.sls b/pillar/node_data/test_ping.sls deleted file mode 100644 index 81cf66004..000000000 --- a/pillar/node_data/test_ping.sls +++ /dev/null @@ -1,23 +0,0 @@ -{% set node_types = {} %} -{% for minionid, test_ping in salt.saltutil.runner('mine.get', tgt='*', fun='test.ping', tgt_type='glob') | dictsort() %} -{% set node_type = minionid.split('_')[1] %} -{% set hostname = minionid.split('_')[0] %} -{% if node_type not in node_types.keys() %} -{% do node_types.update({node_type: {hostname: test_ping}}) %} -{% else %} -{% if hostname not in node_types[node_type] %} -{% do node_types[node_type].update({hostname: test_ping}) %} -{% else %} -{% do node_types[node_type][hostname].update(test_ping) %} -{% endif %} -{% endif %} -{% endfor %} - -node_data: -{% for node_type, values in node_types.items() %} - {{node_type}}: -{% for hostname, test_ping in values.items() %} - {{hostname}}: - test_ping: {{test_ping}} -{% endfor %} -{% endfor %} diff --git a/pillar/top.sls b/pillar/top.sls index 6517fce6d..b6c3b744a 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,8 +2,7 @@ base: '*': - patch.needs_restarting - logrotate - - node_data.ips - - node_data.test_ping + - logstash.nodes '*_eval or *_helixsensor or *_heavynode or *_sensor or *_standalone or *_import': - match: compound diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 6b6adfd6f..05605b96d 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -322,9 +322,9 @@ output.logstash: # The Logstash hosts hosts: -{%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %} +{%- if grains.role in ['so-sensor', 'so-fleet', 'so-node'] %} {%- set LOADBALANCE = ['false'] %} -{%- set node_data = salt['pillar.get']('node_data') %} +{%- set node_data = salt['pillar.get']('logstash:nodes') %} {%- for node_type, node_details in node_data.items() | sort %} {%- if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} {%- for hostname in node_data[node_type].keys() %} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index e5d7228dc..54c8a4273 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -17,12 +17,10 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set LOCALHOSTNAME = salt['grains.get']('host') %} -{% set MAININT = salt['pillar.get']('host:mainint') %} -{% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set MANAGER = salt['grains.get']('master') %} -{% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} +{% from 'filebeat/map.jinja' import EXTRA_HOSTS with context %} {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %} include: @@ -111,7 +109,7 @@ so-filebeat: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} - hostname: so-filebeat - user: root - - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }} + - extra_hosts: {{ EXTRA_HOSTS }} - binds: - /nsm:/nsm:ro - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index 6ae6e7cff..c18fe582c 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -4,3 +4,20 @@ {% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} {% set SO = SODEFAULTS.securityonion_filebeat %} {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} + +{% set role = grains.role %} +{% set EXTRA_HOSTS = [] %} +{% set mainint = salt['pillar.get']('host:mainint') %} +{% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %} +{% if role in ['so-sensor', 'so-fleet', 'so-node' ] %} + {% set node_data = salt['pillar.get']('logstash:nodes') %} + {% for node_type, node_details in node_data.items() | sort %} + {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} + {% for hostname in node_data[node_type].keys() %} + {% do EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %} + {% endfor %} + {% endif %} + {% endfor %} +{% else %} + {% do EXTRA_HOSTS.append({grains.host:localhostip}) %} +{% endif %} diff --git a/salt/salt/etc/minion.d/mine_functions.conf b/salt/salt/etc/minion.d/mine_functions.conf index 023ba0308..42a543841 100644 --- a/salt/salt/etc/minion.d/mine_functions.conf +++ b/salt/salt/etc/minion.d/mine_functions.conf @@ -1,4 +1,3 @@ mine_functions: - test.ping: [] network.ip_addrs: - interface: {{ pillar.host.mainint }}