From e9b12632495db01340aff639fb8f1ff01526f8e1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 May 2024 16:32:43 -0400 Subject: [PATCH 1/7] orchestate searchnode deployment --- salt/manager/tools/sbin/so-minion | 11 +++++++---- salt/orch/deploy_searchnode.sls | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 salt/orch/deploy_searchnode.sls diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 34e069ece..8a34ddca0 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -613,11 +613,14 @@ function updateMineAndApplyStates() { if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then # calls so-common and set_minionid sets MINIONID to local minion id set_minionid - salt $MINIONID state.apply elasticsearch queue=True --async - salt $MINIONID state.apply soc queue=True --async + #salt $MINIONID state.apply elasticsearch queue=True --async + # salt $MINIONID state.apply soc queue=True --async + # $MINIONID is the minion id of the manager and $MINION_ID is the target node or the node being configured + salt-run state.orch orch.deploy_searchnode pillar="{'setup': {'manager': $MINIONID, 'searchnode': $MINION_ID }}" + else + # run this async so the cli doesn't wait for a return + salt "$MINION_ID" state.highstate --async queue=True fi - # run this async so the cli doesn't wait for a return - salt "$MINION_ID" state.highstate --async queue=True } function setupMinionFiles() { diff --git a/salt/orch/deploy_searchnode.sls b/salt/orch/deploy_searchnode.sls new file mode 100644 index 000000000..a70e7bdbf --- /dev/null +++ b/salt/orch/deploy_searchnode.sls @@ -0,0 +1,16 @@ +{% set MANAGER = salt['pillar.get']('setup:manager') %} +{% set SEARCHNODE = salt['pillar.get']('setup:searchnode') %} + +manager_run_es_soc: + salt.state: + - tgt: {{ MANAGER }} + - sls: + - elasticsearch + - soc + +searchnode_run_highstate: + salt.state: + - tgt: {{ TARGET }} + - highstate: True + - require: + - salt: manager_run_es_soc From 29298770422c02d34ee30853d08649c0c334bf78 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 May 2024 16:37:54 -0400 Subject: [PATCH 2/7] fix var --- salt/orch/deploy_searchnode.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/orch/deploy_searchnode.sls b/salt/orch/deploy_searchnode.sls index a70e7bdbf..f36f02511 100644 --- a/salt/orch/deploy_searchnode.sls +++ b/salt/orch/deploy_searchnode.sls @@ -10,7 +10,7 @@ manager_run_es_soc: searchnode_run_highstate: salt.state: - - tgt: {{ TARGET }} + - tgt: {{ SEARCHNODE }} - highstate: True - require: - salt: manager_run_es_soc From bbc374b56ea0a324d926184cfb8c666190005756 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 3 May 2024 09:56:52 -0400 Subject: [PATCH 3/7] add logic in orch --- salt/manager/tools/sbin/so-minion | 16 +++++++------- salt/orch/deploy_newnode.sls | 36 +++++++++++++++++++++++++++++++ salt/orch/deploy_searchnode.sls | 16 -------------- 3 files changed, 44 insertions(+), 24 deletions(-) create mode 100644 salt/orch/deploy_newnode.sls delete mode 100644 salt/orch/deploy_searchnode.sls diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 8a34ddca0..3f8adfa31 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -607,20 +607,20 @@ function updateMineAndApplyStates() { # tell the minion to populate the mine with data from mine_functions which is populated during setup # this only needs to happen on non managers since they handle this during setup # and they need to wait for ca creation to update the mine - updateMine - checkMine "network.ip_addrs" + #updateMine + #checkMine "network.ip_addrs" # apply the elasticsearch state to the manager if a new searchnode was added - if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then + #if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then # calls so-common and set_minionid sets MINIONID to local minion id - set_minionid + set_minionid #salt $MINIONID state.apply elasticsearch queue=True --async # salt $MINIONID state.apply soc queue=True --async # $MINIONID is the minion id of the manager and $MINION_ID is the target node or the node being configured - salt-run state.orch orch.deploy_searchnode pillar="{'setup': {'manager': $MINIONID, 'searchnode': $MINION_ID }}" - else + salt-run state.orch orch.deploy_newnode pillar="{'setup': {'manager': $MINIONID, 'newnode': $MINION_ID }}" + #else # run this async so the cli doesn't wait for a return - salt "$MINION_ID" state.highstate --async queue=True - fi + # salt "$MINION_ID" state.highstate --async queue=True + #fi } function setupMinionFiles() { diff --git a/salt/orch/deploy_newnode.sls b/salt/orch/deploy_newnode.sls new file mode 100644 index 000000000..94ed86723 --- /dev/null +++ b/salt/orch/deploy_newnode.sls @@ -0,0 +1,36 @@ +{% set MANAGER = salt['pillar.get']('setup:manager') %} +{% set NEWNODE = salt['pillar.get']('setup:newnode') %} + +{{NEWNODE}}_update_mine: + salt.function: + - name: mine.update + - tgt: {{ NEWNODE }} + - retry: + attempts: 24 + interval: 5 + +{% if NEWNODE.split('_')|last in ['searchnode', 'heavynode'] %} +manager_run_es_soc: + salt.state: + - tgt: {{ MANAGER }} + - sls: + - elasticsearch + - soc + - kwarg: + queue: True + - retry: + attempts: 30 + interval: 10 + - require: + - salt: new_node_update_mine +{% endif %} + +{{NEWNODE}}_run_highstate: + salt.state: + - tgt: {{ NEWNODE }} + - highstate: True + - kwarg: + queue: True + - retry: + attempts: 30 + interval: 10 diff --git a/salt/orch/deploy_searchnode.sls b/salt/orch/deploy_searchnode.sls deleted file mode 100644 index f36f02511..000000000 --- a/salt/orch/deploy_searchnode.sls +++ /dev/null @@ -1,16 +0,0 @@ -{% set MANAGER = salt['pillar.get']('setup:manager') %} -{% set SEARCHNODE = salt['pillar.get']('setup:searchnode') %} - -manager_run_es_soc: - salt.state: - - tgt: {{ MANAGER }} - - sls: - - elasticsearch - - soc - -searchnode_run_highstate: - salt.state: - - tgt: {{ SEARCHNODE }} - - highstate: True - - require: - - salt: manager_run_es_soc From fa3522a2333a25eb8eb63fabbfd52c178c9b466b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 3 May 2024 11:10:21 -0400 Subject: [PATCH 4/7] fix requirement --- salt/orch/deploy_newnode.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/orch/deploy_newnode.sls b/salt/orch/deploy_newnode.sls index 94ed86723..bf4a67574 100644 --- a/salt/orch/deploy_newnode.sls +++ b/salt/orch/deploy_newnode.sls @@ -22,7 +22,7 @@ manager_run_es_soc: attempts: 30 interval: 10 - require: - - salt: new_node_update_mine + - salt: {{NEWNODE}}_update_mine {% endif %} {{NEWNODE}}_run_highstate: From 442a717d75f071fff0c7e7aebc3e8bb50692209d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 3 May 2024 12:08:57 -0400 Subject: [PATCH 5/7] orchit --- salt/orch/deploy_newnode.sls | 23 +++++++++++++++-------- setup/so-setup | 1 - 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/salt/orch/deploy_newnode.sls b/salt/orch/deploy_newnode.sls index bf4a67574..182fff06c 100644 --- a/salt/orch/deploy_newnode.sls +++ b/salt/orch/deploy_newnode.sls @@ -16,11 +16,10 @@ manager_run_es_soc: - sls: - elasticsearch - soc - - kwarg: - queue: True + - queue: True - retry: - attempts: 30 - interval: 10 + attempts: 3 + interval: 60 - require: - salt: {{NEWNODE}}_update_mine {% endif %} @@ -29,8 +28,16 @@ manager_run_es_soc: salt.state: - tgt: {{ NEWNODE }} - highstate: True - - kwarg: - queue: True + - queue: True - retry: - attempts: 30 - interval: 10 + attempts: 5 + interval: 60 + +{{NEWNODE}}_set_highstate_cron: + salt.state: + - tgt: {{ NEWNODE }} + - sls: + - setup.highstate_cron + - queue: True + - onfail: + - salt: {{NEWNODE}}_run_highstate diff --git a/setup/so-setup b/setup/so-setup index 1c3be22bf..8a1879c58 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -808,7 +808,6 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" check_sos_appliance drop_install_options - logCmd "salt-call state.apply setup.highstate_cron --local --file-root=../salt/" verify_setup fi From 3d4fd59a159901d3deb34381fbff98e88d1d953d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 3 May 2024 13:48:51 -0400 Subject: [PATCH 6/7] orchit --- salt/manager/tools/sbin/so-minion | 21 +++------------------ salt/orch/deploy_newnode.sls | 6 +++++- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 3f8adfa31..e31ec87d3 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -426,10 +426,6 @@ function checkMine() { } -function updateMine() { - retry 20 1 "salt '$MINION_ID' mine.update" True -} - function createEVAL() { is_pcaplimit=true pcapspace @@ -604,23 +600,12 @@ function addMinion() { } function updateMineAndApplyStates() { - # tell the minion to populate the mine with data from mine_functions which is populated during setup - # this only needs to happen on non managers since they handle this during setup - # and they need to wait for ca creation to update the mine - #updateMine + #checkMine "network.ip_addrs" - # apply the elasticsearch state to the manager if a new searchnode was added - #if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then - # calls so-common and set_minionid sets MINIONID to local minion id + # calls so-common and set_minionid sets MINIONID to local minion id set_minionid - #salt $MINIONID state.apply elasticsearch queue=True --async - # salt $MINIONID state.apply soc queue=True --async - # $MINIONID is the minion id of the manager and $MINION_ID is the target node or the node being configured + # $MINIONID is the minion id of the manager and $MINION_ID is the target node or the node being configured salt-run state.orch orch.deploy_newnode pillar="{'setup': {'manager': $MINIONID, 'newnode': $MINION_ID }}" - #else - # run this async so the cli doesn't wait for a return - # salt "$MINION_ID" state.highstate --async queue=True - #fi } function setupMinionFiles() { diff --git a/salt/orch/deploy_newnode.sls b/salt/orch/deploy_newnode.sls index 182fff06c..a2e6b147f 100644 --- a/salt/orch/deploy_newnode.sls +++ b/salt/orch/deploy_newnode.sls @@ -1,14 +1,18 @@ {% set MANAGER = salt['pillar.get']('setup:manager') %} {% set NEWNODE = salt['pillar.get']('setup:newnode') %} +# tell the minion to populate the mine with data from mine_functions which is populated during setup +# this only needs to happen on non managers since they handle this during setup +# and they need to wait for ca creation to update the mine {{NEWNODE}}_update_mine: salt.function: - name: mine.update - tgt: {{ NEWNODE }} - retry: - attempts: 24 + attempts: 36 interval: 5 +# we need to prepare the manager for a new searchnode or heavynode {% if NEWNODE.split('_')|last in ['searchnode', 'heavynode'] %} manager_run_es_soc: salt.state: From bdf1b45a07252a03b31287bb2a86114ed59514df Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 3 May 2024 14:54:44 -0400 Subject: [PATCH 7/7] redirect and throw in bg --- salt/manager/tools/sbin/so-minion | 2 +- salt/orch/deploy_newnode.sls | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index e31ec87d3..79eea59fe 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -605,7 +605,7 @@ function updateMineAndApplyStates() { # calls so-common and set_minionid sets MINIONID to local minion id set_minionid # $MINIONID is the minion id of the manager and $MINION_ID is the target node or the node being configured - salt-run state.orch orch.deploy_newnode pillar="{'setup': {'manager': $MINIONID, 'newnode': $MINION_ID }}" + salt-run state.orch orch.deploy_newnode pillar="{'setup': {'manager': $MINIONID, 'newnode': $MINION_ID }}" > /dev/null 2>&1 & } function setupMinionFiles() { diff --git a/salt/orch/deploy_newnode.sls b/salt/orch/deploy_newnode.sls index a2e6b147f..c05a812a3 100644 --- a/salt/orch/deploy_newnode.sls +++ b/salt/orch/deploy_newnode.sls @@ -21,9 +21,6 @@ manager_run_es_soc: - elasticsearch - soc - queue: True - - retry: - attempts: 3 - interval: 60 - require: - salt: {{NEWNODE}}_update_mine {% endif %} @@ -33,15 +30,3 @@ manager_run_es_soc: - tgt: {{ NEWNODE }} - highstate: True - queue: True - - retry: - attempts: 5 - interval: 60 - -{{NEWNODE}}_set_highstate_cron: - salt.state: - - tgt: {{ NEWNODE }} - - sls: - - setup.highstate_cron - - queue: True - - onfail: - - salt: {{NEWNODE}}_run_highstate