From 8f40b66e3b7f04759a6b4c5dc0ef28c0beb6a207 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 24 Mar 2025 19:49:24 -0400 Subject: [PATCH 1/6] update mine instead of failing highstate if no node_data --- pillar/node_data/ips.sls | 4 ++++ salt/mine/update.sls | 7 +++++++ salt/orch/mine_update.sls | 21 +++++++++++++++++++++ salt/reactor/mine_update.sls | 8 ++++++++ salt/salt/master.sls | 1 + salt/top.sls | 30 +++++++++++++++++------------- 6 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 salt/mine/update.sls create mode 100644 salt/orch/mine_update.sls create mode 100644 salt/reactor/mine_update.sls diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls index 5801d36f1..a2528a476 100644 --- a/pillar/node_data/ips.sls +++ b/pillar/node_data/ips.sls @@ -24,6 +24,7 @@ {% endif %} {% endfor %} +{% if node_types %} node_data: {% for node_type, host_values in node_types.items() %} {% for hostname, details in host_values.items() %} @@ -33,3 +34,6 @@ node_data: role: {{node_type}} {% endfor %} {% endfor %} +{% else %} +node_data: False +{% endif %} diff --git a/salt/mine/update.sls b/salt/mine/update.sls new file mode 100644 index 000000000..59e137d2b --- /dev/null +++ b/salt/mine/update.sls @@ -0,0 +1,7 @@ +# This state sends an event to the salt-master event bus +# The event will be caught by the reactor and trigger the mine_update orchestration + +send_mine_update_event: + module.run: + - name: event.send + - tag: salt/minion/{{grains.id}}/mine_update diff --git a/salt/orch/mine_update.sls b/salt/orch/mine_update.sls new file mode 100644 index 000000000..1483c1225 --- /dev/null +++ b/salt/orch/mine_update.sls @@ -0,0 +1,21 @@ +# Get the minion ID from the pillar +{% set MINION_ID = salt['pillar.get']('minion_id') %} + +# Run mine.update on all minions +update_mine_all_minions: + salt.function: + - name: mine.update + - tgt: '*' + - batch: 50 + - retry: + attempts: 3 + interval: 1 + +# Run highstate on the original minion +run_highstate_on_original_minion: + salt.state: + - tgt: {{ MINION_ID }} + - highstate: True + - queue: True + - require: + - salt: update_mine_all_minions diff --git a/salt/reactor/mine_update.sls b/salt/reactor/mine_update.sls new file mode 100644 index 000000000..968987cec --- /dev/null +++ b/salt/reactor/mine_update.sls @@ -0,0 +1,8 @@ +# This reactor triggers the mine_update orchestration when it receives a mine_update event + +trigger_mine_update_orchestration: + runner.state.orchestrate: + - args: + - mods: orch.mine_update + - pillar: + minion_id: {{ data['id'] }} diff --git a/salt/salt/master.sls b/salt/salt/master.sls index cf9f4718c..ddfb9c3e7 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -3,6 +3,7 @@ include: - salt.minion + - salt.master.reactor_config_mine_update hold_salt_master_package: module.run: diff --git a/salt/top.sls b/salt/top.sls index 437c44bf8..552cd1ea7 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -22,7 +22,11 @@ base: - salt.minion-state-apply-test - salt.minion - '* and G@saltversion:{{saltversion}}': + '*_eval or *_manager* or *_standalone or *_import and I@node_data:False': + - match: compound + - mine.update + + '* and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.minion - patch.os.schedule @@ -33,7 +37,7 @@ base: - docker - docker_clean - '*_sensor and G@saltversion:{{saltversion}}': + '*_sensor and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - sensor - ssl @@ -49,7 +53,7 @@ base: - elasticfleet.install_agent_grid - stig - '*_eval and G@saltversion:{{saltversion}}': + '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - sensor @@ -81,7 +85,7 @@ base: - utility - elasticfleet - '*_manager and G@saltversion:{{saltversion}}': + '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - ca @@ -112,7 +116,7 @@ base: - stig - kafka - '*_standalone and G@saltversion:{{saltversion}}': + '*_standalone and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - sensor @@ -148,7 +152,7 @@ base: - stig - kafka - '*_searchnode and G@saltversion:{{saltversion}}': + '*_searchnode and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - firewall - ssl @@ -161,7 +165,7 @@ base: - stig - kafka - '*_managersearch and G@saltversion:{{saltversion}}': + '*_managersearch and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - ca @@ -192,7 +196,7 @@ base: - stig - kafka - '*_heavynode and G@saltversion:{{saltversion}}': + '*_heavynode and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - sensor - ssl @@ -211,7 +215,7 @@ base: - elasticfleet.install_agent_grid - elasticagent - '*_import and G@saltversion:{{saltversion}}': + '*_import and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - sensor @@ -239,7 +243,7 @@ base: - zeek - elasticfleet - '*_receiver and G@saltversion:{{saltversion}}': + '*_receiver and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - ssl - sensoroni @@ -251,7 +255,7 @@ base: - kafka - stig - '*_idh and G@saltversion:{{saltversion}}': + '*_idh and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - ssl - sensoroni @@ -260,7 +264,7 @@ base: - elasticfleet.install_agent_grid - idh - '*_fleet and G@saltversion:{{saltversion}}': + '*_fleet and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - ssl - sensoroni @@ -272,7 +276,7 @@ base: - elasticfleet.install_agent_grid - schedule - '*_desktop and G@saltversion:{{saltversion}}': + '*_desktop and G@saltversion:{{saltversion}} and not I@node_data:False': - ssl - sensoroni - telegraf From d7e831fbeb55ca42aea7c96651ac6d456443d7ba Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 24 Mar 2025 20:45:35 -0400 Subject: [PATCH 2/6] add mine_update reactor config for master --- salt/salt/master/reactor_config_mine_update.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 salt/salt/master/reactor_config_mine_update.sls diff --git a/salt/salt/master/reactor_config_mine_update.sls b/salt/salt/master/reactor_config_mine_update.sls new file mode 100644 index 000000000..deffbf58d --- /dev/null +++ b/salt/salt/master/reactor_config_mine_update.sls @@ -0,0 +1,13 @@ +reactor_config_hypervisor: + file.managed: + - name: /etc/salt/master.d/reactor_mine_update.conf + - contents: | + reactor: + - 'salt/minion/*/mine_update': + - salt://reactor/mine_update.sls + - group: root + - mode: 644 + - makedirs: True + - watch_in: + - service: salt_master_service + - order: last From 79388af6450a38806f5d14f4ee2a11a0857756b8 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 25 Mar 2025 10:17:43 -0400 Subject: [PATCH 3/6] only managers need node_ips --- pillar/top.sls | 6 +- salt/top.sls | 158 ++++++++++++++++++++++++------------------------- 2 files changed, 84 insertions(+), 80 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index b8d694e23..33b5feb2d 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -24,10 +24,10 @@ base: - firewall.adv_firewall - nginx.soc_nginx - nginx.adv_nginx - - node_data.ips '*_manager or *_managersearch': - match: compound + - node_data.ips {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -90,6 +90,7 @@ base: - soc.license '*_eval': + - node_data.ips - secrets - healthcheck.eval - elasticsearch.index_templates @@ -138,6 +139,7 @@ base: - minions.adv_{{ grains.id }} '*_standalone': + - node_data.ips - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash @@ -260,6 +262,7 @@ base: - soc.license '*_import': + - node_data.ips - secrets - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} @@ -305,6 +308,7 @@ base: - minions.adv_{{ grains.id }} '*_fleet': + - node_data.ips - backup.soc_backup - backup.adv_backup - logstash.nodes diff --git a/salt/top.sls b/salt/top.sls index 552cd1ea7..d33b23932 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -22,11 +22,7 @@ base: - salt.minion-state-apply-test - salt.minion - '*_eval or *_manager* or *_standalone or *_import and I@node_data:False': - - match: compound - - mine.update - - '* and G@saltversion:{{saltversion}} and not I@node_data:False': + '* and G@saltversion:{{saltversion}}': - match: compound - salt.minion - patch.os.schedule @@ -37,21 +33,9 @@ base: - docker - docker_clean - '*_sensor and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_manager* or *_standalone or *_eval or *_import and I@node_data:False': - match: compound - - sensor - - ssl - - sensoroni - - telegraf - - firewall - - nginx - - pcap - - suricata - - healthcheck - - zeek - - strelka - - elasticfleet.install_agent_grid - - stig + - mine.update '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound @@ -85,37 +69,6 @@ base: - utility - elasticfleet - '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - - match: compound - - salt.master - - ca - - ssl - - registry - - nginx - - influxdb - - strelka.manager - - soc - - kratos - - hydra - - firewall - - manager - - sensoroni - - telegraf - - backup.config_backup - - idstools - - suricata.manager - - elasticsearch - - logstash - - redis - - elastic-fleet-package-registry - - kibana - - curator.disabled - - elastalert - - utility - - elasticfleet - - stig - - kafka - '*_standalone and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master @@ -152,16 +105,34 @@ base: - stig - kafka - '*_searchnode and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - - firewall + - salt.master + - ca - ssl - - elasticsearch - - logstash + - registry + - nginx + - influxdb + - strelka.manager + - soc + - kratos + - hydra + - firewall + - manager - sensoroni - telegraf - - nginx - - elasticfleet.install_agent_grid + - backup.config_backup + - idstools + - suricata.manager + - elasticsearch + - logstash + - redis + - elastic-fleet-package-registry + - kibana + - curator.disabled + - elastalert + - utility + - elasticfleet - stig - kafka @@ -196,25 +167,6 @@ base: - stig - kafka - '*_heavynode and G@saltversion:{{saltversion}} and not I@node_data:False': - - match: compound - - sensor - - ssl - - sensoroni - - telegraf - - nginx - - firewall - - elasticsearch - - logstash - - redis - - curator.disabled - - strelka - - pcap - - suricata - - zeek - - elasticfleet.install_agent_grid - - elasticagent - '*_import and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master @@ -243,7 +195,55 @@ base: - zeek - elasticfleet - '*_receiver and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_searchnode and G@saltversion:{{saltversion}}': + - match: compound + - firewall + - ssl + - elasticsearch + - logstash + - sensoroni + - telegraf + - nginx + - elasticfleet.install_agent_grid + - stig + - kafka + + '*_sensor and G@saltversion:{{saltversion}}': + - match: compound + - sensor + - ssl + - sensoroni + - telegraf + - firewall + - nginx + - pcap + - suricata + - healthcheck + - zeek + - strelka + - elasticfleet.install_agent_grid + - stig + + '*_heavynode and G@saltversion:{{saltversion}}': + - match: compound + - sensor + - ssl + - sensoroni + - telegraf + - nginx + - firewall + - elasticsearch + - logstash + - redis + - curator.disabled + - strelka + - pcap + - suricata + - zeek + - elasticfleet.install_agent_grid + - elasticagent + + '*_receiver and G@saltversion:{{saltversion}}': - match: compound - ssl - sensoroni @@ -255,7 +255,7 @@ base: - kafka - stig - '*_idh and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_idh and G@saltversion:{{saltversion}}': - match: compound - ssl - sensoroni @@ -264,7 +264,7 @@ base: - elasticfleet.install_agent_grid - idh - '*_fleet and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_fleet and G@saltversion:{{saltversion}}': - match: compound - ssl - sensoroni @@ -276,7 +276,7 @@ base: - elasticfleet.install_agent_grid - schedule - '*_desktop and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_desktop and G@saltversion:{{saltversion}}': - ssl - sensoroni - telegraf From 55c815cae8c043775442e7b549d361730ce33167 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 25 Mar 2025 19:44:38 -0400 Subject: [PATCH 4/6] simplify highstate rerun when node_data pillar empty --- salt/mine/update.sls | 27 ++++++++++++++++--- salt/orch/mine_update.sls | 21 --------------- salt/reactor/mine_update.sls | 8 ------ salt/salt/master.sls | 1 - .../master/reactor_config_mine_update.sls | 13 --------- salt/top.sls | 13 ++++----- 6 files changed, 30 insertions(+), 53 deletions(-) delete mode 100644 salt/orch/mine_update.sls delete mode 100644 salt/reactor/mine_update.sls delete mode 100644 salt/salt/master/reactor_config_mine_update.sls diff --git a/salt/mine/update.sls b/salt/mine/update.sls index 59e137d2b..d7cdd18d0 100644 --- a/salt/mine/update.sls +++ b/salt/mine/update.sls @@ -1,7 +1,26 @@ # This state sends an event to the salt-master event bus # The event will be caught by the reactor and trigger the mine_update orchestration -send_mine_update_event: - module.run: - - name: event.send - - tag: salt/minion/{{grains.id}}/mine_update +{# may be able to use this method if we can figure out multi state run failure - https://github.com/saltstack/salt/issues/66929 #} +# Get the minion ID from the pillar +{% set MINION_ID = grains.id %} + +# Run mine.update on all minions +mine.update.update_mine_all_minions: + salt.function: + - name: mine.update + - tgt: '*' + - batch: 50 + - retry: + attempts: 3 + interval: 1 + +# Run highstate on the original minion +# we can use concurrent on this highstate because no other highstate would be running when this is called +mine.update.run_highstate_on_{{ MINION_ID }}: + salt.state: + - tgt: {{ MINION_ID }} + - highstate: True + - concurrent: True + - require: + - salt: mine.update.update_mine_all_minions diff --git a/salt/orch/mine_update.sls b/salt/orch/mine_update.sls deleted file mode 100644 index 1483c1225..000000000 --- a/salt/orch/mine_update.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Get the minion ID from the pillar -{% set MINION_ID = salt['pillar.get']('minion_id') %} - -# Run mine.update on all minions -update_mine_all_minions: - salt.function: - - name: mine.update - - tgt: '*' - - batch: 50 - - retry: - attempts: 3 - interval: 1 - -# Run highstate on the original minion -run_highstate_on_original_minion: - salt.state: - - tgt: {{ MINION_ID }} - - highstate: True - - queue: True - - require: - - salt: update_mine_all_minions diff --git a/salt/reactor/mine_update.sls b/salt/reactor/mine_update.sls deleted file mode 100644 index 968987cec..000000000 --- a/salt/reactor/mine_update.sls +++ /dev/null @@ -1,8 +0,0 @@ -# This reactor triggers the mine_update orchestration when it receives a mine_update event - -trigger_mine_update_orchestration: - runner.state.orchestrate: - - args: - - mods: orch.mine_update - - pillar: - minion_id: {{ data['id'] }} diff --git a/salt/salt/master.sls b/salt/salt/master.sls index ddfb9c3e7..cf9f4718c 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -3,7 +3,6 @@ include: - salt.minion - - salt.master.reactor_config_mine_update hold_salt_master_package: module.run: diff --git a/salt/salt/master/reactor_config_mine_update.sls b/salt/salt/master/reactor_config_mine_update.sls deleted file mode 100644 index deffbf58d..000000000 --- a/salt/salt/master/reactor_config_mine_update.sls +++ /dev/null @@ -1,13 +0,0 @@ -reactor_config_hypervisor: - file.managed: - - name: /etc/salt/master.d/reactor_mine_update.conf - - contents: | - reactor: - - 'salt/minion/*/mine_update': - - salt://reactor/mine_update.sls - - group: root - - mode: 644 - - makedirs: True - - watch_in: - - service: salt_master_service - - order: last diff --git a/salt/top.sls b/salt/top.sls index d33b23932..4a6e8e010 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -17,12 +17,17 @@ base: - schedule - logrotate - 'not G@saltversion:{{saltversion}}': + 'I@node_data:False and ( *_manager* or *_eval or *_import or *_standalone )': + - match: compound + - salt.minion + - mine.update + + 'not G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.minion-state-apply-test - salt.minion - '* and G@saltversion:{{saltversion}}': + '* and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.minion - patch.os.schedule @@ -33,10 +38,6 @@ base: - docker - docker_clean - '*_manager* or *_standalone or *_eval or *_import and I@node_data:False': - - match: compound - - mine.update - '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master From 5836bc5bd1daefd71fe2f890bebc13a4b005e857 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 25 Mar 2025 21:58:42 -0400 Subject: [PATCH 5/6] remove require since maybe some failure from mine.update --- salt/mine/update.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/mine/update.sls b/salt/mine/update.sls index d7cdd18d0..03ea76c9e 100644 --- a/salt/mine/update.sls +++ b/salt/mine/update.sls @@ -22,5 +22,3 @@ mine.update.run_highstate_on_{{ MINION_ID }}: - tgt: {{ MINION_ID }} - highstate: True - concurrent: True - - require: - - salt: mine.update.update_mine_all_minions From 667e66bbef008719913d9789b08078c96f7ad01e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 26 Mar 2025 13:56:49 -0400 Subject: [PATCH 6/6] rename mine update and highstate state --- salt/mine/update.sls | 24 -------------------- salt/salt/master/mine_update_highstate.sls | 26 ++++++++++++++++++++++ salt/top.sls | 2 +- 3 files changed, 27 insertions(+), 25 deletions(-) delete mode 100644 salt/mine/update.sls create mode 100644 salt/salt/master/mine_update_highstate.sls diff --git a/salt/mine/update.sls b/salt/mine/update.sls deleted file mode 100644 index 03ea76c9e..000000000 --- a/salt/mine/update.sls +++ /dev/null @@ -1,24 +0,0 @@ -# This state sends an event to the salt-master event bus -# The event will be caught by the reactor and trigger the mine_update orchestration - -{# may be able to use this method if we can figure out multi state run failure - https://github.com/saltstack/salt/issues/66929 #} -# Get the minion ID from the pillar -{% set MINION_ID = grains.id %} - -# Run mine.update on all minions -mine.update.update_mine_all_minions: - salt.function: - - name: mine.update - - tgt: '*' - - batch: 50 - - retry: - attempts: 3 - interval: 1 - -# Run highstate on the original minion -# we can use concurrent on this highstate because no other highstate would be running when this is called -mine.update.run_highstate_on_{{ MINION_ID }}: - salt.state: - - tgt: {{ MINION_ID }} - - highstate: True - - concurrent: True diff --git a/salt/salt/master/mine_update_highstate.sls b/salt/salt/master/mine_update_highstate.sls new file mode 100644 index 000000000..874e6c65b --- /dev/null +++ b/salt/salt/master/mine_update_highstate.sls @@ -0,0 +1,26 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# This state should only be run on managers and should never be run manually + +{% set MINION_ID = grains.id %} + +# Run mine.update on all minions +salt.master.mine_update_highstate.update_mine_all_minions: + salt.function: + - name: mine.update + - tgt: '*' + - batch: 50 + - retry: + attempts: 3 + interval: 1 + +# Run highstate on the original minion +# we can use concurrent on this highstate because no other highstate would be running when this is called +salt.master.mine_update_highstate.run_highstate_on_{{ MINION_ID }}: + salt.state: + - tgt: {{ MINION_ID }} + - highstate: True + - concurrent: True diff --git a/salt/top.sls b/salt/top.sls index 4a6e8e010..ee364b81b 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -20,7 +20,7 @@ base: 'I@node_data:False and ( *_manager* or *_eval or *_import or *_standalone )': - match: compound - salt.minion - - mine.update + - salt.master.mine_update_highstate 'not G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound