From 283939b18a23884f5cd3aa175a701cc99556cef4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Apr 2024 15:36:01 -0400 Subject: [PATCH 1/9] Gather metrics from elastic agent to influx --- salt/manager/init.sls | 20 +++++++++++++ .../manager/tools/sbin/so-elasticagent-status | 10 +++++++ salt/telegraf/scripts/agentstatus.sh | 30 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 salt/manager/tools/sbin/so-elasticagent-status create mode 100644 salt/telegraf/scripts/agentstatus.sh diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 0ff4fa85a..d979482ef 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -27,6 +27,15 @@ repo_log_dir: - user - group +agents_log_dir: + file.directory: + - name: /opt/so/log/agents + - user: root + - group: root + - recurse: + - user + - group + yara_log_dir: file.directory: - name: /opt/so/log/yarasync @@ -101,6 +110,17 @@ so-repo-sync: - hour: '{{ MANAGERMERGED.reposync.hour }}' - minute: '{{ MANAGERMERGED.reposync.minute }}' +so_fleetagent_status: + cron.present: + - name: /usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log 2>&1 + - identifier: so_fleetagent_status + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + socore_own_saltstack: file.directory: - name: /opt/so/saltstack diff --git a/salt/manager/tools/sbin/so-elasticagent-status b/salt/manager/tools/sbin/so-elasticagent-status new file mode 100644 index 000000000..dffd76660 --- /dev/null +++ b/salt/manager/tools/sbin/so-elasticagent-status @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +curl -s -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agent_status" | jq . \ No newline at end of file diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh new file mode 100644 index 000000000..1673e7bd2 --- /dev/null +++ b/salt/telegraf/scripts/agentstatus.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +# if this script isn't already running +if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then + + LOGFILE=$(cat /opt/so/log/agents/agentstatus.log) + ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',') + ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',') + INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',') + OFFLINE=$(cat $LOGFILE | grep -wF offline | awk '{print $2}' | tr -d ',') + UPDATING=$(cat $LOGFILE | grep -wF updating | awk '{print $2}' | tr -d ',') + UNENROLLED=$(cat $LOGFILE | grep -wF unenrolled | awk '{print $2}' | tr -d ',') + OTHER=$(cat $LOGFILE | grep -wF other | awk '{print $2}' | tr -d ',') + EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',') + TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',') + ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',') + ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}') + + echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE" + +fi + +exit 0 From 0efdcfcb52becf95fe6e505c93f07175ea2785fb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 09:36:02 -0400 Subject: [PATCH 2/9] add agentstatus to telegraf --- salt/telegraf/etc/telegraf.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 1c5801645..9ced1317c 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -202,6 +202,16 @@ insecure_skip_verify = true {%- endif %} +# # Get Agent Stats +{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import'] %} +[[inputs.exec]] + commands = [ + "/scripts/agentstatus.sh" + ] + data_format = "influx" + timeout = "15s" +{%- endif %} + # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" # From 0dfde3c9f230f4b76208822cc05d910c0797b9f1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 09:40:14 -0400 Subject: [PATCH 3/9] add agentstatus to telegraf --- salt/telegraf/defaults.yaml | 4 ++++ salt/telegraf/etc/telegraf.conf | 10 ---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index d3718f0b5..3cf2f731d 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -11,6 +11,7 @@ telegraf: quiet: 'false' scripts: eval: + - agentstatus.sh - checkfiles.sh - influxdbsize.sh - lasthighstate.sh @@ -23,6 +24,7 @@ telegraf: - zeekcaptureloss.sh - zeekloss.sh standalone: + - agentstatus.sh - checkfiles.sh - eps.sh - influxdbsize.sh @@ -38,6 +40,7 @@ telegraf: - zeekloss.sh - features.sh manager: + - agentstatus.sh - influxdbsize.sh - lasthighstate.sh - os.sh @@ -46,6 +49,7 @@ telegraf: - sostatus.sh - features.sh managersearch: + - agentstatus.sh - eps.sh - influxdbsize.sh - lasthighstate.sh diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 9ced1317c..1c5801645 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -202,16 +202,6 @@ insecure_skip_verify = true {%- endif %} -# # Get Agent Stats -{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import'] %} -[[inputs.exec]] - commands = [ - "/scripts/agentstatus.sh" - ] - data_format = "influx" - timeout = "15s" -{%- endif %} - # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" # From 513273c8c34868355fc9ca12d34c3dbfe74c8c8c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 09:43:55 -0400 Subject: [PATCH 4/9] add agentstatus to telegraf --- salt/telegraf/enabled.sls | 1 + salt/telegraf/scripts/agentstatus.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index b1fa0c247..8e3bc9fbe 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -56,6 +56,7 @@ so-telegraf: - /opt/so/log/raid:/var/log/raid:ro - /opt/so/log/sostatus:/var/log/sostatus:ro - /opt/so/log/salt:/var/log/salt:ro + - /opt/so/log/agents:/var/log/agents:ro {% if DOCKER.containers['so-telegraf'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-telegraf'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh index 1673e7bd2..ccba213f6 100644 --- a/salt/telegraf/scripts/agentstatus.sh +++ b/salt/telegraf/scripts/agentstatus.sh @@ -10,7 +10,7 @@ # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - LOGFILE=$(cat /opt/so/log/agents/agentstatus.log) + LOGFILE=$(cat /var/log/agents/agentstatus.log) ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',') ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',') INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',') From 333561236588306871f1d4dd605c816cdabd2a53 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 09:54:16 -0400 Subject: [PATCH 5/9] add agentstatus to telegraf --- salt/telegraf/scripts/agentstatus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh index ccba213f6..7ba599e8a 100644 --- a/salt/telegraf/scripts/agentstatus.sh +++ b/salt/telegraf/scripts/agentstatus.sh @@ -10,7 +10,7 @@ # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - LOGFILE=$(cat /var/log/agents/agentstatus.log) + LOGFILE=$(/var/log/agents/agentstatus.log) ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',') ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',') INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',') From 64748b98adf4bdf32a17a2069487d7686c842ea1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 09:56:12 -0400 Subject: [PATCH 6/9] add agentstatus to telegraf --- salt/telegraf/scripts/agentstatus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh index 7ba599e8a..f57fc5f20 100644 --- a/salt/telegraf/scripts/agentstatus.sh +++ b/salt/telegraf/scripts/agentstatus.sh @@ -10,7 +10,7 @@ # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - LOGFILE=$(/var/log/agents/agentstatus.log) + LOGFILE=/var/log/agents/agentstatus.log ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',') ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',') INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',') From 976ddd39820584fda91023d5147b07aa9c722b66 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 10:06:08 -0400 Subject: [PATCH 7/9] add agentstatus to telegraf --- salt/telegraf/scripts/agentstatus.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh index f57fc5f20..a390552fc 100644 --- a/salt/telegraf/scripts/agentstatus.sh +++ b/salt/telegraf/scripts/agentstatus.sh @@ -11,19 +11,23 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then LOGFILE=/var/log/agents/agentstatus.log - ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',') - ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',') - INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',') - OFFLINE=$(cat $LOGFILE | grep -wF offline | awk '{print $2}' | tr -d ',') - UPDATING=$(cat $LOGFILE | grep -wF updating | awk '{print $2}' | tr -d ',') - UNENROLLED=$(cat $LOGFILE | grep -wF unenrolled | awk '{print $2}' | tr -d ',') - OTHER=$(cat $LOGFILE | grep -wF other | awk '{print $2}' | tr -d ',') - EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',') - TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',') - ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',') - ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}') - echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE" + # Check to see if the file is there yet so we don't break install verification since there is a 5 minute delay for this file to show up + if [ -f $LOGFILE ]; then + ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',') + ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',') + INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',') + OFFLINE=$(cat $LOGFILE | grep -wF offline | awk '{print $2}' | tr -d ',') + UPDATING=$(cat $LOGFILE | grep -wF updating | awk '{print $2}' | tr -d ',') + UNENROLLED=$(cat $LOGFILE | grep -wF unenrolled | awk '{print $2}' | tr -d ',') + OTHER=$(cat $LOGFILE | grep -wF other | awk '{print $2}' | tr -d ',') + EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',') + TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',') + ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',') + ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}') + + echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE" + fi fi From 0de1f76139d0b7a26619bca9ae41deace48107e9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 10:26:59 -0400 Subject: [PATCH 8/9] add agent count to reposync --- salt/common/tools/sbin/so-common | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 37adcef99..ebff356e5 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -248,6 +248,11 @@ get_random_value() { head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 } +get_agent_count() { + + AGENTCOUNT=$(/usr/sbin/so-elasticagent-status | grep -wF active | awk '{print $2}') +} + gpg_rpm_import() { if [[ $is_oracle ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then @@ -570,8 +575,9 @@ sync_options() { set_version set_os salt_minion_count + get_agent_count - echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)" + echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)/$AGENTCOUNT" } systemctl_func() { From c31e459c2b4081436fe8ecad695905f9b90cd044 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Apr 2024 11:06:00 -0400 Subject: [PATCH 9/9] Change metrics reporting order --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index ebff356e5..ab3d777ad 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -249,7 +249,7 @@ get_random_value() { } get_agent_count() { - + AGENTCOUNT=$(/usr/sbin/so-elasticagent-status | grep -wF active | awk '{print $2}') } @@ -577,7 +577,7 @@ sync_options() { salt_minion_count get_agent_count - echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)/$AGENTCOUNT" + echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT:$AGENTCOUNT/$(read_feat)" } systemctl_func() {