From 9acaa514cf3f715337ccddc16c5ec906e6024d2f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 Apr 2020 13:06:37 -0400 Subject: [PATCH] monitoring zeek - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/90 --- files/master | 2 + pillar/healthcheck/eval.sls | 2 +- pillar/healthcheck/sensor.sls | 2 +- salt/_beacons/zeek.py | 33 +++++ salt/_modules/healthcheck.py | 48 ++++--- salt/_modules/zeekctl.py | 4 +- .../grafana/grafana_dashboards/eval/eval.json | 126 +++++++++++++++++- .../forward_nodes/sensor.json | 124 +++++++++++++++++ salt/common/init.sls | 4 - salt/healthcheck/init.sls | 3 + salt/reactor/zeek.sls | 18 +++ salt/salt/beacons.sls | 25 ++++ salt/salt/files/beacons.conf.jinja | 8 ++ salt/salt/init.sls | 4 + salt/top.sls | 5 +- 15 files changed, 381 insertions(+), 27 deletions(-) create mode 100644 salt/_beacons/zeek.py create mode 100644 salt/reactor/zeek.sls create mode 100644 salt/salt/beacons.sls create mode 100644 salt/salt/files/beacons.conf.jinja create mode 100644 salt/salt/init.sls diff --git a/files/master b/files/master index f14c4194c..ba107b939 100644 --- a/files/master +++ b/files/master @@ -61,3 +61,5 @@ peer: reactor: - 'so/fleet': - salt://reactor/fleet.sls + - 'salt/beacon/*/zeek/': + - salt://reactor/zeek.sls diff --git a/pillar/healthcheck/eval.sls b/pillar/healthcheck/eval.sls index 09efb7ba7..fbfa54e45 100644 --- a/pillar/healthcheck/eval.sls +++ b/pillar/healthcheck/eval.sls @@ -1,5 +1,5 @@ healthcheck: enabled: False - schedule: 10 + schedule: 60 checks: - zeek diff --git a/pillar/healthcheck/sensor.sls b/pillar/healthcheck/sensor.sls index 09efb7ba7..fbfa54e45 100644 --- a/pillar/healthcheck/sensor.sls +++ b/pillar/healthcheck/sensor.sls @@ -1,5 +1,5 @@ healthcheck: enabled: False - schedule: 10 + schedule: 60 checks: - zeek diff --git a/salt/_beacons/zeek.py b/salt/_beacons/zeek.py new file mode 100644 index 000000000..0db9d3010 --- /dev/null +++ b/salt/_beacons/zeek.py @@ -0,0 +1,33 @@ +import logging + + +def status(): + + cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'" + retval = __salt__['docker.run']('so-zeek', cmd) + logging.debug('zeekctl_module: zeekctl.status retval: %s' % retval) + + return retval + + +def beacon(config): + + retval = [] + + is_enabled = __salt__['healthcheck.is_enabled']() + logging.debug('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled) + + if is_enabled: + zeekstatus = status().lower().split(' ') + logging.debug('zeek_beacon: zeekctl.status: %s' % str(zeekstatus)) + if 'stopped' in zeekstatus or 'crashed' in zeekstatus or 'error' in zeekstatus or 'error:' in zeekstatus: + zeek_restart = True + else: + zeek_restart = False + + __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart)) + retval.append({'zeek_restart': zeek_restart}) + logging.info('zeek_beacon: retval: %s' % str(retval)) + + return retval + diff --git a/salt/_modules/healthcheck.py b/salt/_modules/healthcheck.py index 26e021322..2dafa23d3 100644 --- a/salt/_modules/healthcheck.py +++ b/salt/_modules/healthcheck.py @@ -3,50 +3,60 @@ import logging import sys -allowed_functions = ['zeek'] +allowed_functions = ['is_enabled,zeek'] states_to_apply = [] def apply_states(states=''): calling_func = sys._getframe().f_back.f_code.co_name - logging.debug('healthcheck module: apply_states function caller: %s' % calling_func) + logging.debug('healthcheck_module: apply_states function caller: %s' % calling_func) if not states: states = ','.join(states_to_apply) if states: - logging.info('healthcheck module: apply_states states: %s' % str(states)) + logging.info('healthcheck_module: apply_states states: %s' % str(states)) __salt__['state.apply'](states) -def docker_restart(container): +def docker_stop(container): try: stopdocker = __salt__['docker.rm'](container, 'stop=True') except Exception as e: - logging.error('healthcheck module: %s' % e) + logging.error('healthcheck_module: %s' % e) + + +def is_enabled(): + + if __salt__['pillar.get']('healthcheck:enabled', 'False'): + retval = True + else: + retval = False + + return retval def run(checks=''): retval = [] calling_func = sys._getframe().f_back.f_code.co_name - logging.debug('healthcheck module: run function caller: %s' % calling_func) + logging.debug('healthcheck_module: run function caller: %s' % calling_func) if checks: checks = checks.split(',') else: checks = __salt__['pillar.get']('healthcheck:checks', {}) - logging.debug('healthcheck module: run checks to be run: %s' % str(checks)) + logging.debug('healthcheck_module: run checks to be run: %s' % str(checks)) for check in checks: if check in allowed_functions: retval.append(check) check = getattr(sys.modules[__name__], check) check() else: - logging.warning('healthcheck module: attempted to run function %s' % check) + logging.warning('healthcheck_module: attempted to run function %s' % check) # If you want to apply states at the end of the run, # be sure to append the state name to states_to_apply[] @@ -58,19 +68,23 @@ def run(checks=''): def zeek(): calling_func = sys._getframe().f_back.f_code.co_name - logging.debug('healthcheck module: zeek function caller: %s' % calling_func) + logging.info('healthcheck_module: zeek function caller: %s' % calling_func) + retval = [] retcode = __salt__['zeekctl.status'](verbose=False) - logging.debug('zeekctl.status retcode: %i' % retcode) + logging.info('healthcheck_module: zeekctl.status retcode: %i' % retcode) if retcode: - docker_restart('so-zeek') - states_to_apply.append('zeek') - zeek_restarted = True + zeek_restart = True + if calling_func != 'beacon': + docker_stop('so-zeek') + states_to_apply.append('zeek') else: - zeek_restarted = False + zeek_restart = False - if calling_func == 'execute': + if calling_func == 'execute' and zeek_restart: apply_states() + + retval.append({'zeek_restart': zeek_restart}) - __salt__['telegraf.send']('healthcheck zeek_restarted=%s' % str(zeek_restarted)) - return 'zeek_restarted: %s' % str(zeek_restarted) + __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart)) + return retval diff --git a/salt/_modules/zeekctl.py b/salt/_modules/zeekctl.py index 16d6bf6f1..40f6130e8 100644 --- a/salt/_modules/zeekctl.py +++ b/salt/_modules/zeekctl.py @@ -1,5 +1,7 @@ #!py +import logging + def capstats(interval=10): @@ -140,7 +142,7 @@ def status(verbose=True): retval = __salt__['docker.run']('so-zeek', cmd) if not verbose: retval = __context__['retcode'] - + logging.info('zeekctl_module: zeekctl.status_NOTVERBOSE retval: %s' % retval) return retval diff --git a/salt/common/grafana/grafana_dashboards/eval/eval.json b/salt/common/grafana/grafana_dashboards/eval/eval.json index 28d734ee8..f012bf3e8 100644 --- a/salt/common/grafana/grafana_dashboards/eval/eval.json +++ b/salt/common/grafana/grafana_dashboards/eval/eval.json @@ -516,7 +516,7 @@ } ], "thresholds": "5,10", - "title": "{{ SERVERNAME }} - Zeek Packet Loss", + "title": "{{ SERVERNAME }} -Zeek Packet Loss", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -772,6 +772,130 @@ ], "valueName": "current" }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "healthcheck", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "zeek_restart" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Zeek Restarts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": { "Interrupt": "#70DBED", diff --git a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json index 2c897ba59..914abcb6b 100644 --- a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json +++ b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json @@ -2169,6 +2169,130 @@ ], "valueName": "current" }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "healthcheck", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "zeek_restart" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Zeek Restarts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": { "Buffered": "#6ED0E0", diff --git a/salt/common/init.sls b/salt/common/init.sls index 6e8a3ea65..82a7794d9 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -83,10 +83,6 @@ docker: service.running: - enable: True -salt-minion: - service.running: - - enable: True - # Drop the correct nginx config based on role nginxconfdir: diff --git a/salt/healthcheck/init.sls b/salt/healthcheck/init.sls index 356b8381b..627603099 100644 --- a/salt/healthcheck/init.sls +++ b/salt/healthcheck/init.sls @@ -1,3 +1,6 @@ +### This state isn't used for anything. It was written to handle healthcheck scheduling, +### but we handle that with beacons now. + {% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %} {% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %} {% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %} diff --git a/salt/reactor/zeek.sls b/salt/reactor/zeek.sls new file mode 100644 index 000000000..c22d6f94d --- /dev/null +++ b/salt/reactor/zeek.sls @@ -0,0 +1,18 @@ +#!py + +import logging +import salt.client +local = salt.client.LocalClient() + +def run(): + minionid = data['id'] + zeek_restart = data['zeek_restart'] + + logging.info('zeek_reactor: zeek_need_restarted:%s on:%s' % (zeek_restart, minionid)) + if zeek_restart: + local.cmd(minionid, 'healthcheck.docker_stop', ['so-zeek']) + local.cmd(minionid, 'state.apply', ['zeek']) + +# __salt__['telegraf.send']('healthcheck zeek_restarted=%s' % str(zeek_restarted)) + + return {} diff --git a/salt/salt/beacons.sls b/salt/salt/beacons.sls new file mode 100644 index 000000000..df6198d01 --- /dev/null +++ b/salt/salt/beacons.sls @@ -0,0 +1,25 @@ +{% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %} +{% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %} +{% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %} + +include: + - salt + +{% if CHECKS and ENABLED %} +salt_beacons: + file.managed: + - name: /etc/salt/minion.d/beacons.conf + - source: salt://salt/files/beacons.conf.jinja + - template: jinja + - defaults: + CHECKS: {{ CHECKS }} + SCHEDULE: {{ SCHEDULE }} + - watch_in: + - service: salt_minion_service +{% else %} +salt_beacons: + file.absent: + - name: /etc/salt/minion.d/beacons.conf + - watch_in: + - service: salt_minion_service +{% endif %} diff --git a/salt/salt/files/beacons.conf.jinja b/salt/salt/files/beacons.conf.jinja new file mode 100644 index 000000000..7b2585e74 --- /dev/null +++ b/salt/salt/files/beacons.conf.jinja @@ -0,0 +1,8 @@ +{% if CHECKS -%} +beacons: + {%- for check in CHECKS %} + {{ check }}: + - disable_during_state_run: True + - interval: {{ SCHEDULE }} + {%- endfor %} +{%- endif %} diff --git a/salt/salt/init.sls b/salt/salt/init.sls new file mode 100644 index 000000000..32aaaa47e --- /dev/null +++ b/salt/salt/init.sls @@ -0,0 +1,4 @@ +salt_minion_service: + service.running: + - name: salt-minion + - enable: True diff --git a/salt/top.sls b/salt/top.sls index 10ef82f9a..b9b5697d2 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -12,6 +12,7 @@ base: '*': - patch.os.schedule - motd + - salt '*_helix': - ca @@ -35,7 +36,7 @@ base: - firewall - pcap - suricata - - healthcheck + - salt.beacons {%- if BROVER != 'SURICATA' %} - zeek {%- endif %} @@ -56,7 +57,7 @@ base: - firewall - idstools - auth - - healthcheck + - salt.beacons {%- if FLEETMASTER or FLEETNODE %} - mysql {%- endif %}