From 53b72f87431c6d3d0540e68a8dccfdcf7c849cb2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 Apr 2020 16:54:48 -0400 Subject: [PATCH 1/7] change log levels --- salt/_beacons/zeek.py | 6 +++--- salt/_modules/zeekctl.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/_beacons/zeek.py b/salt/_beacons/zeek.py index 0db9d3010..117c2b401 100644 --- a/salt/_beacons/zeek.py +++ b/salt/_beacons/zeek.py @@ -5,7 +5,7 @@ def status(): cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'" retval = __salt__['docker.run']('so-zeek', cmd) - logging.debug('zeekctl_module: zeekctl.status retval: %s' % retval) + logging.info('zeekctl_module: zeekctl.status retval: %s' % retval) return retval @@ -15,11 +15,11 @@ def beacon(config): retval = [] is_enabled = __salt__['healthcheck.is_enabled']() - logging.debug('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled) + logging.info('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled) if is_enabled: zeekstatus = status().lower().split(' ') - logging.debug('zeek_beacon: zeekctl.status: %s' % str(zeekstatus)) + logging.info('zeek_beacon: zeekctl.status: %s' % str(zeekstatus)) if 'stopped' in zeekstatus or 'crashed' in zeekstatus or 'error' in zeekstatus or 'error:' in zeekstatus: zeek_restart = True else: diff --git a/salt/_modules/zeekctl.py b/salt/_modules/zeekctl.py index 40f6130e8..ab3cb37d3 100644 --- a/salt/_modules/zeekctl.py +++ b/salt/_modules/zeekctl.py @@ -142,7 +142,7 @@ def status(verbose=True): retval = __salt__['docker.run']('so-zeek', cmd) if not verbose: retval = __context__['retcode'] - logging.info('zeekctl_module: zeekctl.status_NOTVERBOSE retval: %s' % retval) + logging.info('zeekctl_module: zeekctl.status retval: %s' % retval) return retval From ec60b7a2d4319de6043d10a1523f8cfce151974c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 Apr 2020 15:22:20 -0400 Subject: [PATCH 2/7] changing back from beacon to scheduled jobs for healthcheck --- files/master | 2 -- pillar/healthcheck/eval.sls | 2 +- pillar/healthcheck/sensor.sls | 2 +- salt/_modules/healthcheck.py | 12 +++++++++--- salt/healthcheck/init.sls | 7 ++----- salt/reactor/zeek.sls | 2 +- salt/top.sls | 4 ++-- 7 files changed, 16 insertions(+), 15 deletions(-) diff --git a/files/master b/files/master index ba107b939..f14c4194c 100644 --- a/files/master +++ b/files/master @@ -61,5 +61,3 @@ peer: reactor: - 'so/fleet': - salt://reactor/fleet.sls - - 'salt/beacon/*/zeek/': - - salt://reactor/zeek.sls diff --git a/pillar/healthcheck/eval.sls b/pillar/healthcheck/eval.sls index fbfa54e45..09efb7ba7 100644 --- a/pillar/healthcheck/eval.sls +++ b/pillar/healthcheck/eval.sls @@ -1,5 +1,5 @@ healthcheck: enabled: False - schedule: 60 + schedule: 10 checks: - zeek diff --git a/pillar/healthcheck/sensor.sls b/pillar/healthcheck/sensor.sls index fbfa54e45..09efb7ba7 100644 --- a/pillar/healthcheck/sensor.sls +++ b/pillar/healthcheck/sensor.sls @@ -1,5 +1,5 @@ healthcheck: enabled: False - schedule: 60 + schedule: 10 checks: - zeek diff --git a/salt/_modules/healthcheck.py b/salt/_modules/healthcheck.py index 2dafa23d3..0a62f0471 100644 --- a/salt/_modules/healthcheck.py +++ b/salt/_modules/healthcheck.py @@ -3,7 +3,7 @@ import logging import sys -allowed_functions = ['is_enabled,zeek'] +allowed_functions = ['is_enabled', 'zeek'] states_to_apply = [] @@ -42,14 +42,14 @@ def run(checks=''): retval = [] calling_func = sys._getframe().f_back.f_code.co_name - logging.debug('healthcheck_module: run function caller: %s' % calling_func) + logging.info('healthcheck_module: run function caller: %s' % calling_func) if checks: checks = checks.split(',') else: checks = __salt__['pillar.get']('healthcheck:checks', {}) - logging.debug('healthcheck_module: run checks to be run: %s' % str(checks)) + logging.info('healthcheck_module: run checks to be run: %s' % str(checks)) for check in checks: if check in allowed_functions: retval.append(check) @@ -65,6 +65,11 @@ def run(checks=''): return retval +def send_event(tag, eventdata): + #__salt__['event.send'](tag, data={'stuff': 'things'}) + __salt__['event.send'](tag, eventdata[0]) + + def zeek(): calling_func = sys._getframe().f_back.f_code.co_name @@ -86,5 +91,6 @@ def zeek(): retval.append({'zeek_restart': zeek_restart}) + send_event('so/healthcheck/zeek', retval) __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart)) return retval diff --git a/salt/healthcheck/init.sls b/salt/healthcheck/init.sls index 627603099..c325bea08 100644 --- a/salt/healthcheck/init.sls +++ b/salt/healthcheck/init.sls @@ -1,9 +1,6 @@ -### This state isn't used for anything. It was written to handle healthcheck scheduling, -### but we handle that with beacons now. - {% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %} {% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %} -{% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %} +{% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 10) %} {% if CHECKS and ENABLED %} {% set STATUS = ['present','enabled'] %} @@ -21,7 +18,7 @@ healthcheck_schedule_{{ STATUS[0] }}: schedule.{{ STATUS[0] }}: - name: healthcheck - function: healthcheck.run - - minutes: {{ SCHEDULE }} + - seconds: {{ SCHEDULE }} healthcheck_schedule_{{ STATUS[1] }}: schedule.{{ STATUS[1] }}: diff --git a/salt/reactor/zeek.sls b/salt/reactor/zeek.sls index c22d6f94d..f2e26b095 100644 --- a/salt/reactor/zeek.sls +++ b/salt/reactor/zeek.sls @@ -6,7 +6,7 @@ local = salt.client.LocalClient() def run(): minionid = data['id'] - zeek_restart = data['zeek_restart'] + zeek_restart = data['data']['zeek_restart'] logging.info('zeek_reactor: zeek_need_restarted:%s on:%s' % (zeek_restart, minionid)) if zeek_restart: diff --git a/salt/top.sls b/salt/top.sls index 7132a4f69..0b08f4c9f 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -37,7 +37,7 @@ base: - firewall - pcap - suricata - - salt.beacons + - healthcheck {%- if BROVER != 'SURICATA' %} - zeek {%- endif %} @@ -58,7 +58,7 @@ base: - firewall - idstools - auth #Shared secrets - - salt.beacons + - healthcheck {%- if FLEETMASTER or FLEETNODE %} - mysql {%- endif %} From 85cfe3244c2e58a57ef638bbe59b61288fede85c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 Apr 2020 19:39:09 -0400 Subject: [PATCH 3/7] rename zeek restarts on grafana dashboard --- salt/common/grafana/grafana_dashboards/eval/eval.json | 2 +- .../common/grafana/grafana_dashboards/forward_nodes/sensor.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/grafana/grafana_dashboards/eval/eval.json b/salt/common/grafana/grafana_dashboards/eval/eval.json index f012bf3e8..9d00efe2f 100644 --- a/salt/common/grafana/grafana_dashboards/eval/eval.json +++ b/salt/common/grafana/grafana_dashboards/eval/eval.json @@ -859,7 +859,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Zeek Restarts", + "title": "Zeek Restarts via Healthcheck", "tooltip": { "shared": true, "sort": 0, diff --git a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json index 914abcb6b..9663dfd79 100644 --- a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json +++ b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json @@ -2256,7 +2256,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Zeek Restarts", + "title": "Zeek Restarts via Healthcheck", "tooltip": { "shared": true, "sort": 0, From 8c3a9ff5a3221d9cb0491ec62ad14107e95633ac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 Apr 2020 19:44:09 -0400 Subject: [PATCH 4/7] change log levels in healthcheck module --- salt/_modules/healthcheck.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/_modules/healthcheck.py b/salt/_modules/healthcheck.py index 0a62f0471..2878b0492 100644 --- a/salt/_modules/healthcheck.py +++ b/salt/_modules/healthcheck.py @@ -42,14 +42,14 @@ def run(checks=''): retval = [] calling_func = sys._getframe().f_back.f_code.co_name - logging.info('healthcheck_module: run function caller: %s' % calling_func) + logging.debug('healthcheck_module: run function caller: %s' % calling_func) if checks: checks = checks.split(',') else: checks = __salt__['pillar.get']('healthcheck:checks', {}) - logging.info('healthcheck_module: run checks to be run: %s' % str(checks)) + logging.debug('healthcheck_module: run checks to be run: %s' % str(checks)) for check in checks: if check in allowed_functions: retval.append(check) @@ -73,11 +73,11 @@ def send_event(tag, eventdata): def zeek(): calling_func = sys._getframe().f_back.f_code.co_name - logging.info('healthcheck_module: zeek function caller: %s' % calling_func) + logging.debug('healthcheck_module: zeek function caller: %s' % calling_func) retval = [] retcode = __salt__['zeekctl.status'](verbose=False) - logging.info('healthcheck_module: zeekctl.status retcode: %i' % retcode) + logging.debug('healthcheck_module: zeekctl.status retcode: %i' % retcode) if retcode: zeek_restart = True if calling_func != 'beacon': From 5411307856ef496942cba8176abf59f8a1662835 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 Apr 2020 19:46:29 -0400 Subject: [PATCH 5/7] change default healthcheck schedule to 300 seconds --- pillar/healthcheck/eval.sls | 2 +- pillar/healthcheck/sensor.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pillar/healthcheck/eval.sls b/pillar/healthcheck/eval.sls index 09efb7ba7..dd1a027e9 100644 --- a/pillar/healthcheck/eval.sls +++ b/pillar/healthcheck/eval.sls @@ -1,5 +1,5 @@ healthcheck: enabled: False - schedule: 10 + schedule: 300 checks: - zeek diff --git a/pillar/healthcheck/sensor.sls b/pillar/healthcheck/sensor.sls index 09efb7ba7..dd1a027e9 100644 --- a/pillar/healthcheck/sensor.sls +++ b/pillar/healthcheck/sensor.sls @@ -1,5 +1,5 @@ healthcheck: enabled: False - schedule: 10 + schedule: 300 checks: - zeek From f1857c61b982494be56f1315acc414cb89f2a17d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 Apr 2020 19:48:21 -0400 Subject: [PATCH 6/7] remove comment from healthcheck module --- salt/_modules/healthcheck.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/_modules/healthcheck.py b/salt/_modules/healthcheck.py index 2878b0492..e5aedff00 100644 --- a/salt/_modules/healthcheck.py +++ b/salt/_modules/healthcheck.py @@ -66,7 +66,6 @@ def run(checks=''): def send_event(tag, eventdata): - #__salt__['event.send'](tag, data={'stuff': 'things'}) __salt__['event.send'](tag, eventdata[0]) From c42f8276e23686b625e8c90c9468e3aa2957c052 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 Apr 2020 19:48:54 -0400 Subject: [PATCH 7/7] change default in healthcheck state --- salt/healthcheck/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/healthcheck/init.sls b/salt/healthcheck/init.sls index c325bea08..94f8028ba 100644 --- a/salt/healthcheck/init.sls +++ b/salt/healthcheck/init.sls @@ -1,6 +1,6 @@ {% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %} {% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %} -{% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 10) %} +{% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 300) %} {% if CHECKS and ENABLED %} {% set STATUS = ['present','enabled'] %}