Merge pull request #14457 from Security-Onion-Solutions/mineimp

prevent manager node type highstate failure from missing network.ip_addrs in mine

pillar/node_data/ips.sls

@@ -24,6 +24,7 @@
 {%   endif %}
 {% endfor %}
 
+{% if node_types %}
 node_data:
 {% for node_type, host_values in node_types.items() %}
 {%   for hostname, details in host_values.items() %}
@@ -33,3 +34,6 @@ node_data:
     role: {{node_type}}
 {%   endfor %}
 {% endfor %}
+{% else %}
+node_data: False
+{% endif %}

pillar/top.sls

@@ -24,10 +24,10 @@ base:
     - firewall.adv_firewall
     - nginx.soc_nginx
     - nginx.adv_nginx
-    - node_data.ips
 
   '*_manager or *_managersearch':
     - match: compound
+    - node_data.ips
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
@@ -90,6 +90,7 @@ base:
     - soc.license
 
   '*_eval':
+    - node_data.ips
     - secrets
     - healthcheck.eval
     - elasticsearch.index_templates
@@ -138,6 +139,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
+    - node_data.ips
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -260,6 +262,7 @@ base:
     - soc.license
 
   '*_import':
+    - node_data.ips
     - secrets
     - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -305,6 +308,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_fleet':
+    - node_data.ips
     - backup.soc_backup
     - backup.adv_backup
     - logstash.nodes

salt/salt/master/mine_update_highstate.sls

@@ -0,0 +1,26 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# This state should only be run on managers and should never be run manually
+
+{% set MINION_ID = grains.id %}
+
+# Run mine.update on all minions
+salt.master.mine_update_highstate.update_mine_all_minions:
+  salt.function:
+    - name: mine.update
+    - tgt: '*'
+    - batch: 50
+    - retry:
+        attempts: 3
+        interval: 1
+
+# Run highstate on the original minion
+# we can use concurrent on this highstate because no other highstate would be running when this is called
+salt.master.mine_update_highstate.run_highstate_on_{{ MINION_ID }}:
+  salt.state:
+    - tgt: {{ MINION_ID }}
+    - highstate: True
+    - concurrent: True

salt/top.sls

@@ -17,12 +17,17 @@ base:
     - schedule
     - logrotate
 
-  'not G@saltversion:{{saltversion}}':
+  'I@node_data:False and ( *_manager* or *_eval or *_import or *_standalone )':
+    - match: compound
+    - salt.minion
+    - salt.master.mine_update_highstate
+
+  'not G@saltversion:{{saltversion}} and not I@node_data:False':
     - match: compound
     - salt.minion-state-apply-test
     - salt.minion
 
-  '* and G@saltversion:{{saltversion}}':
+  '* and G@saltversion:{{saltversion}} and not I@node_data:False':
     - match: compound
     - salt.minion
     - patch.os.schedule
@@ -33,6 +38,177 @@ base:
     - docker
     - docker_clean
 
+  '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False':
+    - match: compound
+    - salt.master
+    - sensor
+    - ca
+    - ssl
+    - registry
+    - manager
+    - backup.config_backup
+    - nginx
+    - influxdb
+    - soc
+    - kratos
+    - hydra
+    - sensoroni
+    - telegraf
+    - firewall
+    - idstools
+    - suricata.manager
+    - healthcheck
+    - elasticsearch
+    - elastic-fleet-package-registry
+    - kibana
+    - pcap
+    - suricata
+    - zeek
+    - strelka
+    - curator.disabled
+    - elastalert
+    - utility
+    - elasticfleet
+
+  '*_standalone and G@saltversion:{{saltversion}} and not I@node_data:False':
+    - match: compound
+    - salt.master
+    - sensor
+    - ca
+    - ssl
+    - registry
+    - manager
+    - backup.config_backup
+    - nginx
+    - influxdb
+    - soc
+    - kratos
+    - hydra
+    - firewall
+    - sensoroni
+    - telegraf
+    - idstools
+    - suricata.manager
+    - healthcheck
+    - elasticsearch
+    - logstash
+    - redis
+    - elastic-fleet-package-registry
+    - kibana
+    - pcap
+    - suricata
+    - zeek
+    - strelka
+    - curator.disabled
+    - elastalert
+    - utility
+    - elasticfleet
+    - stig
+    - kafka
+
+  '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False':
+    - match: compound
+    - salt.master
+    - ca
+    - ssl
+    - registry
+    - nginx
+    - influxdb
+    - strelka.manager
+    - soc
+    - kratos
+    - hydra
+    - firewall
+    - manager
+    - sensoroni
+    - telegraf
+    - backup.config_backup
+    - idstools
+    - suricata.manager
+    - elasticsearch
+    - logstash
+    - redis
+    - elastic-fleet-package-registry
+    - kibana
+    - curator.disabled
+    - elastalert
+    - utility
+    - elasticfleet
+    - stig
+    - kafka
+
+  '*_managersearch and G@saltversion:{{saltversion}} and not I@node_data:False':
+    - match: compound
+    - salt.master
+    - ca
+    - ssl
+    - registry
+    - nginx
+    - influxdb
+    - strelka.manager
+    - soc
+    - kratos
+    - hydra
+    - firewall
+    - manager
+    - sensoroni
+    - telegraf
+    - backup.config_backup
+    - idstools
+    - suricata.manager
+    - elasticsearch
+    - logstash
+    - redis
+    - curator.disabled
+    - elastic-fleet-package-registry
+    - kibana
+    - elastalert
+    - utility
+    - elasticfleet
+    - stig
+    - kafka
+
+  '*_import and G@saltversion:{{saltversion}} and not I@node_data:False':
+    - match: compound
+    - salt.master
+    - sensor
+    - ca
+    - ssl
+    - registry
+    - manager
+    - nginx
+    - influxdb
+    - strelka.manager
+    - soc
+    - kratos
+    - hydra
+    - sensoroni
+    - telegraf
+    - firewall
+    - idstools
+    - suricata.manager
+    - pcap
+    - elasticsearch
+    - elastic-fleet-package-registry
+    - kibana
+    - utility
+    - suricata
+    - zeek
+    - elasticfleet
+
+  '*_searchnode and G@saltversion:{{saltversion}}':
+    - match: compound
+    - firewall
+    - ssl
+    - elasticsearch
+    - logstash
+    - sensoroni
+    - telegraf
+    - nginx
+    - elasticfleet.install_agent_grid
+    - stig
+    - kafka
+
   '*_sensor and G@saltversion:{{saltversion}}':
     - match: compound
     - sensor
@@ -49,149 +225,6 @@ base:
     - elasticfleet.install_agent_grid
     - stig
 
-  '*_eval and G@saltversion:{{saltversion}}':
-    - match: compound
-    - salt.master
-    - sensor
-    - ca
-    - ssl
-    - registry
-    - manager
-    - backup.config_backup
-    - nginx
-    - influxdb
-    - soc
-    - kratos
-    - hydra
-    - sensoroni
-    - telegraf
-    - firewall
-    - idstools
-    - suricata.manager
-    - healthcheck
-    - elasticsearch
-    - elastic-fleet-package-registry
-    - kibana
-    - pcap
-    - suricata
-    - zeek
-    - strelka
-    - curator.disabled
-    - elastalert
-    - utility
-    - elasticfleet
-
-  '*_manager and G@saltversion:{{saltversion}}':
-    - match: compound
-    - salt.master
-    - ca
-    - ssl
-    - registry
-    - nginx
-    - influxdb
-    - strelka.manager
-    - soc
-    - kratos
-    - hydra
-    - firewall
-    - manager
-    - sensoroni
-    - telegraf
-    - backup.config_backup
-    - idstools
-    - suricata.manager
-    - elasticsearch
-    - logstash
-    - redis
-    - elastic-fleet-package-registry
-    - kibana
-    - curator.disabled
-    - elastalert
-    - utility
-    - elasticfleet
-    - stig
-    - kafka
-
-  '*_standalone and G@saltversion:{{saltversion}}':
-    - match: compound
-    - salt.master
-    - sensor
-    - ca
-    - ssl
-    - registry
-    - manager
-    - backup.config_backup
-    - nginx
-    - influxdb
-    - soc
-    - kratos
-    - hydra
-    - firewall
-    - sensoroni
-    - telegraf
-    - idstools
-    - suricata.manager
-    - healthcheck
-    - elasticsearch
-    - logstash
-    - redis
-    - elastic-fleet-package-registry
-    - kibana
-    - pcap
-    - suricata
-    - zeek
-    - strelka
-    - curator.disabled
-    - elastalert
-    - utility
-    - elasticfleet
-    - stig
-    - kafka
-
-  '*_searchnode and G@saltversion:{{saltversion}}':
-    - match: compound
-    - firewall
-    - ssl
-    - elasticsearch
-    - logstash
-    - sensoroni
-    - telegraf
-    - nginx
-    - elasticfleet.install_agent_grid
-    - stig
-    - kafka
-
-  '*_managersearch and G@saltversion:{{saltversion}}':
-    - match: compound
-    - salt.master
-    - ca
-    - ssl
-    - registry
-    - nginx
-    - influxdb
-    - strelka.manager
-    - soc
-    - kratos
-    - hydra
-    - firewall
-    - manager
-    - sensoroni
-    - telegraf
-    - backup.config_backup
-    - idstools
-    - suricata.manager
-    - elasticsearch
-    - logstash
-    - redis
-    - curator.disabled
-    - elastic-fleet-package-registry
-    - kibana
-    - elastalert
-    - utility
-    - elasticfleet
-    - stig
-    - kafka
-
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound
     - sensor
@@ -211,34 +244,6 @@ base:
     - elasticfleet.install_agent_grid
     - elasticagent
 
-  '*_import and G@saltversion:{{saltversion}}':
-    - match: compound
-    - salt.master
-    - sensor
-    - ca
-    - ssl
-    - registry
-    - manager
-    - nginx
-    - influxdb
-    - strelka.manager
-    - soc
-    - kratos
-    - hydra
-    - sensoroni
-    - telegraf
-    - firewall
-    - idstools
-    - suricata.manager
-    - pcap
-    - elasticsearch
-    - elastic-fleet-package-registry
-    - kibana
-    - utility
-    - suricata
-    - zeek
-    - elasticfleet
-
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound
     - ssl