From 9797a152189d3f53e8a83f9952f0aff4fba314e9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 14 Oct 2021 17:23:18 -0400 Subject: [PATCH 01/16] Fix issue with 'so-user delete' resetting all user roles - note that this function is not technically supported or published since it's not intended for production use --- salt/common/tools/sbin/so-user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 015a28c9f..5a52a9d59 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -445,7 +445,7 @@ function deleteUser() { rolesTmpFile="${socRolesFile}.tmp" createFile "$rolesTmpFile" "$soUID" "$soGID" - grep -v "$id" "$socRolesFile" > "$rolesTmpFile" + grep -v "$identityId" "$socRolesFile" > "$rolesTmpFile" mv "$rolesTmpFile" "$socRolesFile" } From 1800ec4570f2045f16d10f88116df5b3985aa4c8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 15 Oct 2021 09:25:44 -0400 Subject: [PATCH 02/16] Upgrade to Elastalert 2 v2.2.2 --- salt/common/tools/sbin/so-elastalert-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastalert-test b/salt/common/tools/sbin/so-elastalert-test index 51d00e6fa..2d3106ab6 100755 --- a/salt/common/tools/sbin/so-elastalert-test +++ b/salt/common/tools/sbin/so-elastalert-test @@ -70,7 +70,7 @@ do done docker_exec(){ - CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/config/elastalert_config.yaml $OPTIONS" + CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/elastalert/config.yaml $OPTIONS" if [ "${RESULTS_TO_LOG,,}" = "y" ] ; then $CMD > "$FILE_SAVE_LOCATION" else From 032373187cc577f721d2b8b95b426da6a98fc00a Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 15 Oct 2021 16:02:53 +0000 Subject: [PATCH 03/16] Allow setting ES index patterns for SOC in pillar --- salt/soc/files/soc/soc.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index e33ea406b..02128fd3c 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -26,7 +26,7 @@ {%- set ES_USER = '' %} {%- set ES_PASS = '' %} {%- endif %} - +{%- set ES_INDEX_PATTERNS = salt['pillar.get']('soc:es_index_patterns', '*:so-*') %} { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", "server": { @@ -57,6 +57,7 @@ {%- endif %} "username": "{{ ES_USER }}", "password": "{{ ES_PASS }}", + "index": "{{ ES_INDEX_PATTERNS }}", "cacheMs": {{ ES_FIELDCAPS_CACHE }}, "verifyCert": false, "timeoutMs": {{ API_TIMEOUT }} From 8feeff97b5869397ad902736c7768087d3474537 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 15 Oct 2021 16:19:19 +0000 Subject: [PATCH 04/16] Add EG index pattern during setup (if enabled) --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index af66896c6..3f6a2b136 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1645,6 +1645,7 @@ manager_pillar() { printf '%s\n'\ "soc:"\ " endgamehost: '$ENDGAMEHOST'"\ + " es_index_patterns: '*:so-*, *:endgame-*'"\ "" >> "$pillar_file" fi } From 8de8d5815516a6e14575e699c8a8e5f366c2699f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 15 Oct 2021 13:27:08 -0400 Subject: [PATCH 05/16] Upgrade to ES 7.15.1 --- salt/kibana/files/saved_objects.ndjson.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/saved_objects.ndjson.jinja b/salt/kibana/files/saved_objects.ndjson.jinja index abc722a9e..9cf037b78 100644 --- a/salt/kibana/files/saved_objects.ndjson.jinja +++ b/salt/kibana/files/saved_objects.ndjson.jinja @@ -462,7 +462,7 @@ {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} {% endraw -%} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.0","id":"7.15.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.1","id":"7.15.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {% raw -%} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} From a9f6c84d7cf5d251366fb1fc349ac98bab4f329a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 15 Oct 2021 14:17:54 -0400 Subject: [PATCH 06/16] Add Steno Tuning Options --- salt/pcap/files/config | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/salt/pcap/files/config b/salt/pcap/files/config index 900234bc1..24f9a579e 100644 --- a/salt/pcap/files/config +++ b/salt/pcap/files/config @@ -1,20 +1,23 @@ -{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} -{%- set diskfreepercentage = salt['pillar.get']('steno:diskfreepercentage', 10) %} -{%- set maxfiles = salt['pillar.get']('steno:maxfiles', 30000) %} - - +{%- set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %} +{%- set DISKFREEPERCENTAGE = salt['pillar.get']('steno:diskfreepercentage', 10) %} +{%- set MAXFILES = salt['pillar.get']('steno:maxfiles', 30000) %} +{%- set BLOCKS = salt['pillar.get']('steno:blocks', 2048) %} +{%- set FILEMB = salt['pillar.get']('steno:filemb', 4096) %} +{%- set AIOPS = salt['pillar.get']('steno:aiops', 128) %} +{%- set THREADS = salt['pillar.get']('steno:threads', 1) %} { "Threads": [ - { "PacketsDirectory": "/nsm/pcap" - , "IndexDirectory": "/nsm/pcapindex" - , "MaxDirectoryFiles": {{ maxfiles }} - , "DiskFreePercentage": {{ diskfreepercentage }} - } + { "PacketsDirectory": "/nsm/pcap", "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} } + {%- if THREADS > 1 %} + {%- for i in range(2,THREADS+1) %} + , { "PacketsDirectory": "/nsm/pcap" , "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} } + {%- endfor %} + {%- endif %} ] , "StenotypePath": "/usr/bin/stenotype" - , "Interface": "{{ interface }}" + , "Interface": "{{ INTERFACE }}" , "Port": 1234 , "Host": "127.0.0.1" - , "Flags": ["-v", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] + , "Flags": ["-v", "--blocks={{ BLOCKS }}", "--preallocate_file_mb={{ FILEMB }}", "--aiops={{ AIOPS }}", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] , "CertPath": "/etc/stenographer/certs" } From d0a6dafc8bc49d22797c86cb060317eaf11bb278 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 15 Oct 2021 17:09:59 -0400 Subject: [PATCH 07/16] Add TI module --- salt/common/tools/sbin/so-filebeat-module-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 401f54289..14a3ebed2 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -54,7 +54,7 @@ PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_ if [[ "$PIPELINES" -lt 5 ]]; then echo "Setting up ingest pipeline(s)" - for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler + for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler do echo "Loading $MODULE" docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML From bb36fc1ed8e594378b8d1758209f7895afa56f05 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 15 Oct 2021 17:16:38 -0400 Subject: [PATCH 08/16] Add TI module defaults --- salt/filebeat/thirdpartydefaults.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index 112ed6d6c..3be8bb901 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -244,6 +244,23 @@ third_party_filebeat: var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9501 + threatintel: + abuseurl: + enabled: false + abusemalware: + enabled: false + misp: + enabled: false + malwarebazaar: + enabled: false + otx: + enabled: false + anomali: + enabled: false + anomalithreatstream: + enabled: false + recordedfuture: + enabled: false zscaler: zia: enabled: false From b9a3d3a6a96f415498f1c8b1ca0918d4d74a30d6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 19 Oct 2021 11:14:02 -0400 Subject: [PATCH 09/16] Fix Steno Math for PL --- salt/telegraf/scripts/stenoloss.sh | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index 028637e16..cd09b952f 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -19,25 +19,30 @@ THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then - TSFILE=/var/log/telegraf/laststenodrop.log - if [ -f "$TSFILE" ]; then - LASTTS=$(cat $TSFILE) - else - LASTTS=0 + CHECKIT=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2|wc -l) + STENOGREP=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2) + + declare RESULT=($STENOGREP) + + CURRENT_PACKETS=$(echo ${RESULT[9]} | awk -F'=' '{print $2 }') + CURRENT_DROPS=$(echo ${RESULT[12]} | awk -F'=' '{print $2 }') + PREVIOUS_PACKETS=$(echo ${RESULT[23]} | awk -F'=' '{print $2 }') + PREVIOUS_DROPS=$(echo ${RESULT[26]} | awk -F'=' '{print $2 }') + + DROPPED=$((CURRENT_DROPS - PREVIOUS_DROPS)) + TOTAL_CURRENT=$((CURRENT_PACKETS + CURRENT_DROPS)) + TOTAL_PAST=$((PREVIOUS_PACKETS + PREVIOUS_DROPS)) + TOTAL=$((TOTAL_CURRENT - TOTAL_PAST)) + + if [ $CHECKIT == 2 ]; then + if [ $DROPPED == 0 ]; then + echo "stenodrop drop=$DROPPED" + else + LOSS=$(echo "4 k $DROPPED $TOTAL / 100 * p" | dc) + echo "stenodrop drop=$loss" + fi fi - - # Get the data - LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) - CURRENTTS=$(echo $LOGLINE | awk '{print $1}') - - if [[ "$CURRENTTS" != "$LASTTS" ]]; then - DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') - echo $CURRENTTS > $TSFILE - else - DROP=0 - fi - - echo "stenodrop drop=$DROP" + else exit 0 fi \ No newline at end of file From 64f25961b0b785b80031e53b7da54af0c2ac3b59 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 19 Oct 2021 11:15:58 -0400 Subject: [PATCH 10/16] Fix Steno Math for PL --- salt/telegraf/scripts/stenoloss.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index cd09b952f..09b7ddb70 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -39,7 +39,7 @@ if [ ! "$THEGREP" ]; then echo "stenodrop drop=$DROPPED" else LOSS=$(echo "4 k $DROPPED $TOTAL / 100 * p" | dc) - echo "stenodrop drop=$loss" + echo "stenodrop drop=$LOSS" fi fi From 9453ed7fa11fbf0c5908105d987615a8fbab0240 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 19 Oct 2021 13:01:40 -0400 Subject: [PATCH 11/16] Remove space to allow pattern(s) to be correctly interpreted --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 3f6a2b136..f4d08e9a9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1645,7 +1645,7 @@ manager_pillar() { printf '%s\n'\ "soc:"\ " endgamehost: '$ENDGAMEHOST'"\ - " es_index_patterns: '*:so-*, *:endgame-*'"\ + " es_index_patterns: '*:so-*,*:endgame-*'"\ "" >> "$pillar_file" fi } From 2f8bb5a2a6729c13d9e2233dfe81821dd8a1a580 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 19 Oct 2021 16:04:10 -0400 Subject: [PATCH 12/16] Fix Docker-created corruption of SOC user roles file --- salt/common/tools/sbin/so-user | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 5a52a9d59..9bf36cf99 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -182,6 +182,10 @@ function ensureRoleFileExists() { echo "Database file does not exist yet, installation is likely not yet complete." fi + if [[ -d "$socRolesFile" ]]; then + echo "Removing invalid roles directory created by Docker" + rm -fr "$socRolesFile" + fi mv "${rolesTmpFile}" "${socRolesFile}" fi } From b496810b63fc6d0d8bc3dba0092b402554a6ec92 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 14:46:47 -0400 Subject: [PATCH 13/16] add redis and logstash input plugins to telegraf --- salt/telegraf/etc/telegraf.conf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index b04b1549d..71fc610b4 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -639,6 +639,21 @@ insecure_skip_verify = true {% endif %} +{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.logstash]] + url = "http://localhost:9600" + collect = ["pipelines"] + {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + username = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:pass') }}" + {% endif %} +{%- endif %} + +{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.redis]] + servers = "tcp://localhost:5000" +{%- endif %} + # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" From 7ecfb55b7026e6858512de04b25c6c137c4b42c4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 14:50:50 -0400 Subject: [PATCH 14/16] fix pillar call --- salt/telegraf/etc/telegraf.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 71fc610b4..dac41123e 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -644,8 +644,8 @@ url = "http://localhost:9600" collect = ["pipelines"] {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} - username = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:user') }}" - password = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:pass') }}" + username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" {% endif %} {%- endif %} From 8619af59ccee3a5ed24aede22a37bcf1de1e0c55 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 15:02:33 -0400 Subject: [PATCH 15/16] servers to list format --- salt/telegraf/etc/telegraf.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index dac41123e..b92ba9f33 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -651,7 +651,7 @@ {% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} [[inputs.redis]] - servers = "tcp://localhost:5000" + servers = ["tcp://localhost:5000"] {%- endif %} # From adffb1180045d483df72dfb989ef05ae6ded41cc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 15:39:21 -0400 Subject: [PATCH 16/16] fix redis port --- salt/telegraf/etc/telegraf.conf | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index b92ba9f33..59d806fe0 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -639,21 +639,6 @@ insecure_skip_verify = true {% endif %} -{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} -[[inputs.logstash]] - url = "http://localhost:9600" - collect = ["pipelines"] - {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} - username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" - password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" - {% endif %} -{%- endif %} - -{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} -[[inputs.redis]] - servers = ["tcp://localhost:5000"] -{%- endif %} - # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" @@ -688,9 +673,22 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false +{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.logstash]] + url = "http://localhost:9600" + collect = ["pipelines"] + {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" + {% endif %} +{%- endif %} + +{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.redis]] + servers = ["tcp://localhost:6379"] +{%- endif %} # # Read metrics from one or more commands that can output to stdout - [[inputs.exec]] commands = [ "/scripts/sostatus.sh"