changes to healthcheck. specify mount points for telegraf

pillar/healthcheck/eval.sls

@@ -0,0 +1,5 @@
+healthcheck:
+  enabled: False
+  schedule: 10
+  checks:
+    - zeek

pillar/healthcheck/sensor.sls

@@ -0,0 +1,5 @@
+healthcheck:
+  enabled: False
+  schedule: 10
+  checks:
+    - zeek

pillar/top.sls

@@ -35,6 +35,7 @@ base:
     - auth
     - logstash
     - logstash.eval
+    - healthcheck.eval
     - minions.{{ grains.id }}
 
   '*_node':

salt/_modules/healthcheck.py

@@ -1,11 +1,42 @@
 #!py
 
 import logging
+import sys
+
+allowed_functions = ['zeek']
+states_to_apply = []
+
+
+def apply_states():
+
+  if states_to_apply:
+    states = ','.join(states_to_apply)
+    __salt__['state.apply'](states)
+
 
 def docker_restart(container, state):
-  stopdocker = __salt__['docker.rm'](container, 'force=True')
-  __salt__['state.apply'](state)
   
+  try:
+    stopdocker = __salt__['docker.rm'](container, 'stop=True')
+  except Exception as e:
+    logging.error('healthcheck module: %s' % e)
+  
+
+def run(checks):
+  if checks:
+    checks = checks.split(',')
+  else:  
+    checks = __salt__['pillar.get']('healthcheck:checks', {})
+  
+  for check in checks:
+    if check in allowed_functions:
+      check = getattr(sys.modules[__name__], check)
+      check()
+    else:
+      logging.warning('healthcheck module: attempted to run function %s' % check)
+
+
+  return checks
 
 
 def zeek():
@@ -13,10 +44,13 @@ def zeek():
   retcode = __salt__['zeekctl.status'](verbose=False)
   logging.info('zeekctl.status retcode: %i' % retcode)
   if retcode:
-    docker_restart('so-zeek', 'zeek')
+    docker_restart('so-zeek')
+    states_to_apply.append('zeek')
     zeek_restarted = True
   else:
     zeek_restarted = False
 
   __salt__['telegraf.send']('healthcheck zeek_restarted=%s' % str(zeek_restarted))
   return 'zeek_restarted: %s' % str(zeek_restarted)
+
+apply_states()

salt/common/telegraf/etc/telegraf.conf

@@ -498,10 +498,10 @@
 [[inputs.disk]]
   ## By default stats will be gathered for all mount points.
   ## Set mount_points will restrict the stats to only the specified mount points.
-  # mount_points = ["/"]
+  mount_points = ["/","/nsm"]
 
   ## Ignore mount points by filesystem type.
-  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"]
+  #ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"]
 
 
 # Read metrics about disk IO by device

salt/healthcheck/init.sls

@@ -0,0 +1,26 @@
+{% set CHECKS = salt['pillar.get']('healthcheck:checks', {} %}
+{% set ENABLED = salt['pillar.get']('healthcheck:enabled', False %}
+{% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30 %}
+
+{% if CHECKS and ENABLED %}
+  {% set STATUS = ['present','enabled'] %}
+{% else %}
+  {% set STATUS = ['absent','disabled'] %}
+nohealthchecks:
+  test.configurable_test_state:
+    - name: nohealthchecks
+    - changes: True
+    - result: False
+    - comment: No checks are enabled for the healthcheck schedule
+    - warnings: Add checks to the healcheck:checks pillar
+{% endif %}
+
+healthcheck_schedule_{{STATUS[0]}}:
+  schedule.{{STATUS[0]}}:
+    - name: healthcheck
+    - function: healthcheck.run
+    - minutes: {{ SCHEDULE }}
+
+healthcheck_schedule_{{STATUS[1]}}:
+  schedule.{{STATUS[1]}}:
+    - name: healthcheck