From 449703744280bf5df6272681c6950e62df997d6f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Nov 2022 20:03:54 -0500 Subject: [PATCH 1/6] Use bg:True to send cmd to background --- salt/soc/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 8356bd1d8..94cad69c8 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -84,7 +84,8 @@ salt-relay: cmd.run: - env: - SOC_PIPE: /opt/sensoroni/salt.pipe - - name: '/opt/so/saltstack/default/salt/soc/files/bin/salt-relay.sh >> /opt/so/log/soc/salt-relay.log 2>&1 &' + - name: '/opt/so/saltstack/default/salt/soc/files/bin/salt-relay.sh >> /opt/so/log/soc/salt-relay.log 2>&1' + - bg: True - unless: ps -ef | grep salt-relay | grep -v grep so-soc: From 7cd5d625d121bd14934803e43ab1709131cdf4d0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Nov 2022 20:45:50 -0500 Subject: [PATCH 2/6] temporarily remove salt-pipe for debug purposes --- salt/soc/init.sls | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 94cad69c8..28453fbf9 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -80,13 +80,7 @@ socusersroles: - require: - sls: manager.sync_es_users -salt-relay: - cmd.run: - - env: - - SOC_PIPE: /opt/sensoroni/salt.pipe - - name: '/opt/so/saltstack/default/salt/soc/files/bin/salt-relay.sh >> /opt/so/log/soc/salt-relay.log 2>&1' - - bg: True - - unless: ps -ef | grep salt-relay | grep -v grep + so-soc: docker_container.running: From c572848ece40326cfe45e45aecd1c11cb849020e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Nov 2022 08:06:24 -0500 Subject: [PATCH 3/6] temporarily remove filecheck for debug purposes --- salt/strelka/init.sls | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index e3477dd9e..d0c48fd55 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -134,12 +134,6 @@ filecheck_script: - group: 939 - mode: 755 -filecheck_run: - cmd.run: - - name: 'python3 /opt/so/conf/strelka/filecheck' - - bg: True - - runas: socore - - unless: ps -ef | grep filecheck | grep -v grep filcheck_history_clean: cron.present: From 0ffef75d7baf9da289de330caabbb8871c00f60e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Nov 2022 09:50:41 -0500 Subject: [PATCH 4/6] Move background jobs to cron --- salt/soc/defaults.yaml | 2 +- salt/soc/files/bin/salt-relay.sh | 2 +- salt/soc/init.sls | 12 ++++++++++-- salt/strelka/init.sls | 5 +++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 278a02342..0f41c32f2 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -83,7 +83,7 @@ soc: bucket: telegraf verifyCert: false salt: - saltPipe: /opt/sensoroni/salt.pipe + saltPipe: /opt/sensoroni/salt/pipe sostatus: refreshIntervalMs: 30000 offlineThresholdMs: 900000 diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index 514f1e616..c4d0d0037 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -6,7 +6,7 @@ PIPE_OWNER=${PIPE_OWNER:-socore} PIPE_GROUP=${PIPE_GROUP:-socore} -SOC_PIPE=${SOC_PIPE_REQUEST:-/opt/so/conf/soc/salt.pipe} +SOC_PIPE=${SOC_PIPE_REQUEST:-/opt/so/conf/soc/salt/pipe} function log() { echo "$(date) | $1" diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 28453fbf9..64ebdc671 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -27,6 +27,12 @@ soclogdir: - group: 939 - makedirs: True +socsaltdir: + file.directory: + - name: /opt/so/conf/soc/salt + - user: 939 + - group: 939 + - makedirs: True socconfig: file.managed: @@ -80,7 +86,9 @@ socusersroles: - require: - sls: manager.sync_es_users - +salt-relay: + cron.present: + - name: 'ps -ef | grep salt-relay | grep -v grep || /opt/so/saltstack/default/salt/soc/files/bin/salt-relay.sh >> /opt/so/log/soc/salt-relay.log 2>&1 &' so-soc: docker_container.running: @@ -96,7 +104,7 @@ so-soc: - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - - /opt/so/conf/soc/salt.pipe:/opt/sensoroni/salt.pipe:rw + - /opt/so/conf/soc/salt:/opt/sensoroni/salt:rw - /opt/so/saltstack:/opt/so/saltstack:rw {%- if salt['pillar.get']('nodestab', {}) %} - extra_hosts: diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index d0c48fd55..0706cda66 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -134,6 +134,11 @@ filecheck_script: - group: 939 - mode: 755 +filecheck_run: + cron.present: + - name: 'ps -ef | grep filecheck | grep -v grep || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' + - user: socore + - minute: 9 filcheck_history_clean: cron.present: From 7f7e5474edba1e16623fa6919eeef3547e0e4dd2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Nov 2022 10:43:05 -0500 Subject: [PATCH 5/6] Add more logging for filecheck monitoring, and ensure scripts are accessible to salt-relay --- salt/soc/files/bin/salt-relay.sh | 1 + salt/strelka/filecheck/filecheck | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index c4d0d0037..238e8ec29 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -7,6 +7,7 @@ PIPE_OWNER=${PIPE_OWNER:-socore} PIPE_GROUP=${PIPE_GROUP:-socore} SOC_PIPE=${SOC_PIPE_REQUEST:-/opt/so/conf/soc/salt/pipe} +PATH=${PATH}:/usr/sbin function log() { echo "$(date) | $1" diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index 3d498ce62..35bcc7f79 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -65,6 +65,8 @@ if __name__ == "__main__": event_handler =CreatedEventHandler() observer = Observer() + + logging.info("Starting filecheck") observer.schedule(event_handler, extract_path, recursive=True) observer.start() try: @@ -72,4 +74,6 @@ if __name__ == "__main__": time.sleep(1) except KeyboardInterrupt: observer.stop() - observer.join() \ No newline at end of file + observer.join() + + logging.info("Exiting filecheck") \ No newline at end of file From ed9aa5b73f81b9535ddff6cce02a13664c48a829 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Nov 2022 10:48:53 -0500 Subject: [PATCH 6/6] Ensure filecheck is up by checking every minute --- salt/strelka/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 0706cda66..155126f91 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -138,7 +138,6 @@ filecheck_run: cron.present: - name: 'ps -ef | grep filecheck | grep -v grep || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' - user: socore - - minute: 9 filcheck_history_clean: cron.present: