Merge pull request #11563 from Security-Onion-Solutions/minechanges

Minechanges

pillar/logstash/nodes.sls

@@ -7,19 +7,23 @@
     tgt_type='compound') | dictsort()
 %}
 
-{%   set hostname = cached_grains[minionid]['host'] %}
+# only add a node to the pillar if it returned an ip from the mine
-{%   set node_type = minionid.split('_')[1] %}
+{%   if ip | length > 0%}
-{%   if node_type not in node_types.keys() %}
+{%     set hostname = cached_grains[minionid]['host'] %}
-{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     set node_type = minionid.split('_')[1] %}
-{%   else %}
+{%     if node_type not in node_types.keys() %}
-{%     if hostname not in node_types[node_type] %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
-{%       do node_types[node_type].update({hostname: ip[0]}) %}
 {%     else %}
-{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
 {%     endif %}
 {%   endif %}
 {% endfor %}
 
+
 logstash:
   nodes:
 {% for node_type, values in node_types.items() %}

pillar/node_data/ips.sls

@@ -4,18 +4,22 @@
 {%   set hostname = minionid.split('_')[0] %}
 {%   set node_type = minionid.split('_')[1] %}
 {%   set is_alive = False %}
-{%   if minionid in manage_alived.keys() %}
+
-{%     if ip[0] == manage_alived[minionid] %}
+# only add a node to the pillar if it returned an ip from the mine
-{%       set is_alive = True %}
+{%   if ip | length > 0%}
+{%     if minionid in manage_alived.keys() %}
+{%       if ip[0] == manage_alived[minionid] %}
+{%         set is_alive = True %}
+{%       endif %}
 {%     endif %}
-{%   endif %}
+{%     if node_type not in node_types.keys() %}
-{%   if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %}
-{%     do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %}
-{%   else %}
-{%     if hostname not in node_types[node_type] %}
-{%       do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %}
 {%     else %}
-{%       do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %}
+{%       endif %}
 {%     endif %}
 {%   endif %}
 {% endfor %}

salt/salt/engines/master/checkmine.py

@@ -9,6 +9,10 @@ log = logging.getLogger(__name__)
 local = salt.client.LocalClient()
 
 def start(interval=60):
+    def mine_delete(minion, func):
+      log.warning('checkmine engine: deleting mine function %s for %s' % (func, minion))
+      local.cmd(minion, 'mine.delete', [func])
+
     def mine_flush(minion):
         log.warning('checkmine engine: flushing mine cache for %s' % minion)
         local.cmd(minion, 'mine.flush')
@@ -21,28 +25,35 @@ def start(interval=60):
     cachedir = __opts__['cachedir']
     while True:
         log.debug('checkmine engine: checking which minions are alive')
-        manage_alived = __salt__['saltutil.runner']('manage.alived', show_ip=True)
+        manage_alived = __salt__['saltutil.runner']('manage.alived', show_ip=False)
         log.debug('checkmine engine: alive minions: %s' % ' , '.join(manage_alived))
 
         for minion in manage_alived:
             mine_path = os.path.join(cachedir, 'minions', minion, 'mine.p')
-            mine_size = os.path.getsize(mine_path)
+            # it is possible that a minion is alive, but hasn't created a mine file yet
-            log.debug('checkmine engine: minion: %s mine_size: %i' % (minion, mine_size))
+            try:
-            # For some reason the mine file can be corrupt and only be 1 byte in size
+                mine_size = os.path.getsize(mine_path)
-            if mine_size == 1:
+                log.debug('checkmine engine: minion: %s mine_size: %i' % (minion, mine_size))
-                log.error('checkmine engine: found %s to be 1 byte' % mine_path)
+                # For some reason the mine file can be corrupt and only be 1 byte in size
-                mine_flush(minion)
+                if mine_size == 1:
-                mine_update(minion)
+                    log.error('checkmine engine: found %s to be 1 byte' % mine_path)
-            # Update the mine if the ip in the mine doesn't match returned from manage.alived
-            else:
-                network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs')
-                mine_ip = network_ip_addrs[minion][0]
-                log.debug('checkmine engine: minion: %s mine_ip: %s' % (minion, mine_ip))
-                manage_alived_ip = manage_alived[minion]
-                log.debug('checkmine engine: minion: %s managed_alived_ip: %s' % (minion, manage_alived_ip))
-                if mine_ip != manage_alived_ip:
-                    log.error('checkmine engine: found minion %s has manage_alived_ip %s but a mine_ip of %s' % (minion, manage_alived_ip, mine_ip))
                     mine_flush(minion)
                     mine_update(minion)
+                    continue
+            except FileNotFoundError:
+                log.warning('checkmine engine: minion: %s %s does not exist' % (minion, mine_path))
+                mine_flush(minion)
+                mine_update(minion)
+                continue
+
+            # Update the mine if the ip in the mine doesn't match returned from manage.alived
+            network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs')
+            try:
+                mine_ip = network_ip_addrs[minion][0]
+                log.debug('checkmine engine: found minion %s has mine_ip: %s' % (minion, mine_ip))
+            except IndexError:
+                log.error('checkmine engine: found minion %s does\'t have a mine_ip' % (minion))
+                mine_delete(minion, 'network.ip_addrs')
+                mine_update(minion)
 
         sleep(interval)