monitoring zeek - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/90

files/master

@@ -61,3 +61,5 @@ peer:
 reactor:
   - 'so/fleet':
     - salt://reactor/fleet.sls
+  - 'salt/beacon/*/zeek/':
+    - salt://reactor/zeek.sls

pillar/healthcheck/eval.sls

@@ -1,5 +1,5 @@
 healthcheck:
   enabled: False
-  schedule: 10
+  schedule: 60
   checks:
     - zeek

pillar/healthcheck/sensor.sls

@@ -1,5 +1,5 @@
 healthcheck:
   enabled: False
-  schedule: 10
+  schedule: 60
   checks:
     - zeek

salt/_beacons/zeek.py

@@ -0,0 +1,33 @@
+import logging
+
+
+def status():
+
+ cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'"
+ retval = __salt__['docker.run']('so-zeek', cmd)
+ logging.debug('zeekctl_module: zeekctl.status retval: %s' % retval)
+
+ return retval
+
+
+def beacon(config):
+
+  retval = []
+
+  is_enabled = __salt__['healthcheck.is_enabled']()
+  logging.debug('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled)
+
+  if is_enabled:
+    zeekstatus = status().lower().split(' ')
+    logging.debug('zeek_beacon: zeekctl.status: %s' % str(zeekstatus))
+    if 'stopped' in zeekstatus or 'crashed' in zeekstatus or 'error' in zeekstatus or 'error:' in zeekstatus:
+     zeek_restart = True
+    else:
+     zeek_restart = False
+
+    __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart))
+    retval.append({'zeek_restart': zeek_restart})
+    logging.info('zeek_beacon: retval: %s' % str(retval))
+
+  return retval
+

salt/_modules/healthcheck.py

@@ -3,50 +3,60 @@
 import logging
 import sys
 
-allowed_functions = ['zeek']
+allowed_functions = ['is_enabled,zeek']
 states_to_apply = []
 
 
 def apply_states(states=''):
 
   calling_func = sys._getframe().f_back.f_code.co_name
-  logging.debug('healthcheck module: apply_states function caller: %s' % calling_func)
+  logging.debug('healthcheck_module: apply_states function caller: %s' % calling_func)
 
   if not states:
     states = ','.join(states_to_apply)
  
   if states: 
-    logging.info('healthcheck module: apply_states states: %s' % str(states))
+    logging.info('healthcheck_module: apply_states states: %s' % str(states))
     __salt__['state.apply'](states)
 
 
-def docker_restart(container):
+def docker_stop(container):
   
   try:
     stopdocker = __salt__['docker.rm'](container, 'stop=True')
   except Exception as e:
-    logging.error('healthcheck module: %s' % e)
+    logging.error('healthcheck_module: %s' % e)
+
+
+def is_enabled():
+  
+  if __salt__['pillar.get']('healthcheck:enabled', 'False'):
+    retval = True
+  else:
+    retval = False
+  
+  return retval
   
 
 def run(checks=''):
   
   retval = []
   calling_func = sys._getframe().f_back.f_code.co_name
-  logging.debug('healthcheck module: run function caller: %s' % calling_func)
+  logging.debug('healthcheck_module: run function caller: %s' % calling_func)
   
   if checks:
     checks = checks.split(',')
   else:  
     checks = __salt__['pillar.get']('healthcheck:checks', {})
   
-  logging.debug('healthcheck module: run checks to be run: %s' % str(checks))
+  logging.debug('healthcheck_module: run checks to be run: %s' % str(checks))
   for check in checks:
     if check in allowed_functions:
       retval.append(check)
       check = getattr(sys.modules[__name__], check)
       check()
     else:
-      logging.warning('healthcheck module: attempted to run function %s' % check)
+      logging.warning('healthcheck_module: attempted to run function %s' % check)
 
   # If you want to apply states at the end of the run,
   # be sure to append the state name to states_to_apply[]
@@ -58,19 +68,23 @@ def run(checks=''):
 def zeek():
 
   calling_func = sys._getframe().f_back.f_code.co_name
-  logging.debug('healthcheck module: zeek function caller: %s' % calling_func)
+  logging.info('healthcheck_module: zeek function caller: %s' % calling_func)
+  retval = []
 
   retcode = __salt__['zeekctl.status'](verbose=False)
-  logging.debug('zeekctl.status retcode: %i' % retcode)
+  logging.info('healthcheck_module: zeekctl.status retcode: %i' % retcode)
   if retcode:
-    docker_restart('so-zeek')
+    zeek_restart = True
-    states_to_apply.append('zeek')
+    if calling_func != 'beacon':
-    zeek_restarted = True
+      docker_stop('so-zeek')
+      states_to_apply.append('zeek')
   else:
-    zeek_restarted = False
+    zeek_restart = False
 
-  if calling_func == 'execute':
+  if calling_func == 'execute' and zeek_restart:
     apply_states()
+  
+  retval.append({'zeek_restart': zeek_restart})
 
-  __salt__['telegraf.send']('healthcheck zeek_restarted=%s' % str(zeek_restarted))
+  __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart))
-  return 'zeek_restarted: %s' % str(zeek_restarted)
+  return retval

salt/_modules/zeekctl.py

@@ -1,5 +1,7 @@
 #!py
 
+import logging
+
 
 def capstats(interval=10):
 
@@ -140,7 +142,7 @@ def status(verbose=True):
  retval = __salt__['docker.run']('so-zeek', cmd)
  if not verbose:
    retval = __context__['retcode']
-
+   logging.info('zeekctl_module: zeekctl.status_NOTVERBOSE retval: %s' % retval)
  return retval
 
 

salt/common/grafana/grafana_dashboards/eval/eval.json

@@ -516,7 +516,7 @@
         }
       ],
       "thresholds": "5,10",
-      "title": "{{ SERVERNAME }} - Zeek Packet Loss",
+      "title": "{{ SERVERNAME }} -Zeek Packet Loss",
       "type": "singlestat",
       "valueFontSize": "80%",
       "valueMaps": [
@@ -772,6 +772,130 @@
       ],
       "valueName": "current"
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "InfluxDB",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 29
+      },
+      "hiddenSeries": false,
+      "id": 37,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "healthcheck",
+          "orderByTime": "ASC",
+          "policy": "autogen",
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "zeek_restart"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "last"
+              }
+            ]
+          ],
+          "tags": [
+            {
+              "key": "host",
+              "operator": "=",
+              "value": "{{ SERVERNAME }}"
+            }
+          ]
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Zeek Restarts",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "aliasColors": {
         "Interrupt": "#70DBED",

salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json

@@ -2169,6 +2169,130 @@
       ],
       "valueName": "current"
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "InfluxDB",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 29
+      },
+      "hiddenSeries": false,
+      "id": 37,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "healthcheck",
+          "orderByTime": "ASC",
+          "policy": "autogen",
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "zeek_restart"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "last"
+              }
+            ]
+          ],
+          "tags": [
+            {
+              "key": "host",
+              "operator": "=",
+              "value": "{{ SERVERNAME }}"
+            }
+          ]
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Zeek Restarts",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "aliasColors": {
         "Buffered": "#6ED0E0",

salt/common/init.sls

@@ -83,10 +83,6 @@ docker:
   service.running:
     - enable: True
 
-salt-minion:
-  service.running:
-    - enable: True
-
 # Drop the correct nginx config based on role
 
 nginxconfdir:

salt/healthcheck/init.sls

@@ -1,3 +1,6 @@
+### This state isn't used for anything. It was written to handle healthcheck scheduling,
+### but we handle that with beacons now.
+
 {% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %}
 {% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %}
 {% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %}

salt/reactor/zeek.sls

@@ -0,0 +1,18 @@
+#!py
+
+import logging
+import salt.client
+local = salt.client.LocalClient()
+
+def run():
+  minionid = data['id']
+  zeek_restart = data['zeek_restart']
+  
+  logging.info('zeek_reactor: zeek_need_restarted:%s on:%s' % (zeek_restart, minionid))
+  if zeek_restart:
+    local.cmd(minionid, 'healthcheck.docker_stop', ['so-zeek'])
+    local.cmd(minionid, 'state.apply', ['zeek'])
+
+#    __salt__['telegraf.send']('healthcheck zeek_restarted=%s' % str(zeek_restarted))
+
+  return {}

salt/salt/beacons.sls

@@ -0,0 +1,25 @@
+{% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %}
+{% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %}
+{% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %}
+
+include:
+  - salt
+
+{% if CHECKS and ENABLED %}
+salt_beacons:
+  file.managed:
+    - name: /etc/salt/minion.d/beacons.conf
+    - source: salt://salt/files/beacons.conf.jinja
+    - template: jinja
+    - defaults:
+        CHECKS: {{ CHECKS }}
+        SCHEDULE: {{ SCHEDULE }}
+    - watch_in: 
+      - service: salt_minion_service
+{% else %}
+salt_beacons:
+  file.absent:
+    - name: /etc/salt/minion.d/beacons.conf
+    - watch_in: 
+      - service: salt_minion_service
+{% endif %}

salt/salt/files/beacons.conf.jinja

@@ -0,0 +1,8 @@
+{% if CHECKS -%}
+beacons:
+  {%- for check in CHECKS %}
+  {{ check }}:
+    - disable_during_state_run: True
+    - interval: {{ SCHEDULE }}
+  {%- endfor %}
+{%- endif %}

salt/salt/init.sls

@@ -0,0 +1,4 @@
+salt_minion_service:
+  service.running:
+    - name: salt-minion
+    - enable: True

salt/top.sls

@@ -12,6 +12,7 @@ base:
   '*':
     - patch.os.schedule
     - motd
+    - salt
 
   '*_helix':
     - ca
@@ -35,7 +36,7 @@ base:
     - firewall
     - pcap
     - suricata
-    - healthcheck
+    - salt.beacons
     {%- if BROVER != 'SURICATA' %}
     - zeek
     {%- endif %}
@@ -56,7 +57,7 @@ base:
     - firewall
     - idstools
     - auth
-    - healthcheck
+    - salt.beacons
     {%- if FLEETMASTER or FLEETNODE %}
     - mysql
     {%- endif %}