diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 1cc7ceed0..0711260b5 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -32,17 +32,16 @@ tgrafetsdir: - name: /opt/so/conf/telegraf/scripts - makedirs: True -tgrafsyncscripts: - file.recurse: - - name: /opt/so/conf/telegraf/scripts +{% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +tgraf_sync_script_{{script}}: + file.managed: + - name: /opt/so/conf/telegraf/scripts/{{script}} - user: root - group: 939 - - file_mode: 770 + - mode: 770 - template: jinja - - source: salt://telegraf/scripts - {% if GLOBALS.md_engine == 'SURICATA' %} - - exclude_pat: zeekcaptureloss.sh - {% endif %} + - source: salt://telegraf/scripts/{{script}} +{% endfor %} telegraf_sbin: file.recurse: diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 63d437763..36ef679f0 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -9,3 +9,82 @@ telegraf: flush_jitter: '0s' debug: 'false' quiet: 'false' + scripts: + eval: + - beatseps.sh + - checkfiles.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + standalone: + - beatseps.sh + - checkfiles.sh + - eps.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + manager: + - beatseps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + managersearch: + - beatseps.sh + - eps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + import: + - sostatus.sh + sensor: + - beatseps.sh + - checkfiles.sh + - oldpcap.sh + - raid.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + heavynode: + - beatseps.sh + - checkfiles.sh + - eps.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + idh: + - sostatus.sh + searchnode: + - beatseps.sh + - eps.sh + - raid.sh + - sostatus.sh + receiver: + - beatseps.sh + - eps.sh + - raid.sh + - redis.sh + - sostatus.sh + fleet: + - sostatus.sh + desktop: [] diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index 209c85fb0..598587e17 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} include: @@ -67,8 +68,10 @@ so-telegraf: {% endif %} - watch: - file: tgrafconf - - file: tgrafsyncscripts - file: node_config + {% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + - file: tgraf_sync_script_{{script}} + {% endfor %} - require: - file: tgrafconf - file: node_config diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 1a6cdc311..45b1283e0 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -193,7 +193,7 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true -{%- elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode'] %} +{%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = false @@ -244,6 +244,8 @@ {%- endif %} # # Read metrics from one or more commands that can output to stdout +{%- if 'sostatus.sh' in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %} [[inputs.exec]] commands = [ "/scripts/sostatus.sh" @@ -251,122 +253,26 @@ data_format = "influx" timeout = "15s" interval = "60s" +{%- endif %} -# ## Commands array -{% if grains['role'] in ['so-manager'] %} +{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %} [[inputs.exec]] commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" +{%- for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + "/scripts/{{script}}"{% if not loop.last %},{% endif %} +{%- endfor %} ] data_format = "influx" ## Timeout for each command to complete. timeout = "15s" -{% elif grains['role'] in ['so-managersearch'] %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] in ['so-searchnode', 'so-receiver'] %} -[[inputs.exec]] - commands = [ - "/scripts/eps.sh", - "/scripts/raid.sh", - {% if grains.role == 'so-receiver' %} - "/scripts/redis.sh", - {% endif %} - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] == 'so-sensor' %} -[[inputs.exec]] - commands = [ - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-heavynode' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-standalone' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-eval' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% endif %} +{%- endif %} {%- if salt['pillar.get']('healthcheck:enabled', False) %} [[inputs.file]] files = ["/host/nsm/zeek/logs/zeek_restart.log"] data_format = "influx" {%- endif %} + [[inputs.file]] files = ["/etc/telegraf/node_config.json"] name_override = "node_config" diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index f1412d3ac..e6d3460d6 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -2,6 +2,16 @@ or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} - + +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %} {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %} + +{% if GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode'] %} +{% from 'zeek/config.map.jinja' import ZEEKMERGED %} +{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} +{% if GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% endif %} +{% endif %} diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index e0c8758f2..f2c3fcd2d 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -5,16 +5,18 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - - # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running +{%- from 'zeek/config.map.jinja' import ZEEKMERGED %} if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} +{%- if ZEEKMERGED.config.node.pins %} + WORKERS={{ ZEEKMERGED.config.node.pins | length }} +{%- else %} + WORKERS={{ ZEEKMERGED.config.node.lb_procs }} +{%- endif %} ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then WORKERS=1 diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index a688ea2a3..1550c66cb 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -42,4 +42,21 @@ telegraf: global: True advanced: True helpLink: telegraf.html - \ No newline at end of file + scripts: + eval: &telegrafscripts + description: List of input.exec scripts to run for this node type. The script must be present in salt/telegraf/scripts. + forcedType: "[]string" + multiline: True + advanced: True + helpLink: telegraf.html + standalone: *telegrafscripts + manager: *telegrafscripts + managersearch: *telegrafscripts + import: *telegrafscripts + sensor: *telegrafscripts + heavynode: *telegrafscripts + idh: *telegrafscripts + searchnode: *telegrafscripts + receiver: *telegrafscripts + fleet: *telegrafscripts + desktop: *telegrafscripts