diff --git a/salt/common/tools/sbin/so-elastalert-test b/salt/common/tools/sbin/so-elastalert-test index 51d00e6fa..2d3106ab6 100755 --- a/salt/common/tools/sbin/so-elastalert-test +++ b/salt/common/tools/sbin/so-elastalert-test @@ -70,7 +70,7 @@ do done docker_exec(){ - CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/config/elastalert_config.yaml $OPTIONS" + CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/elastalert/config.yaml $OPTIONS" if [ "${RESULTS_TO_LOG,,}" = "y" ] ; then $CMD > "$FILE_SAVE_LOCATION" else diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 401f54289..14a3ebed2 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -54,7 +54,7 @@ PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_ if [[ "$PIPELINES" -lt 5 ]]; then echo "Setting up ingest pipeline(s)" - for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler + for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler do echo "Loading $MODULE" docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 015a28c9f..9bf36cf99 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -182,6 +182,10 @@ function ensureRoleFileExists() { echo "Database file does not exist yet, installation is likely not yet complete." fi + if [[ -d "$socRolesFile" ]]; then + echo "Removing invalid roles directory created by Docker" + rm -fr "$socRolesFile" + fi mv "${rolesTmpFile}" "${socRolesFile}" fi } @@ -445,7 +449,7 @@ function deleteUser() { rolesTmpFile="${socRolesFile}.tmp" createFile "$rolesTmpFile" "$soUID" "$soGID" - grep -v "$id" "$socRolesFile" > "$rolesTmpFile" + grep -v "$identityId" "$socRolesFile" > "$rolesTmpFile" mv "$rolesTmpFile" "$socRolesFile" } diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index 112ed6d6c..3be8bb901 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -244,6 +244,23 @@ third_party_filebeat: var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9501 + threatintel: + abuseurl: + enabled: false + abusemalware: + enabled: false + misp: + enabled: false + malwarebazaar: + enabled: false + otx: + enabled: false + anomali: + enabled: false + anomalithreatstream: + enabled: false + recordedfuture: + enabled: false zscaler: zia: enabled: false diff --git a/salt/kibana/files/saved_objects.ndjson.jinja b/salt/kibana/files/saved_objects.ndjson.jinja index abc722a9e..9cf037b78 100644 --- a/salt/kibana/files/saved_objects.ndjson.jinja +++ b/salt/kibana/files/saved_objects.ndjson.jinja @@ -462,7 +462,7 @@ {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} {% endraw -%} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.0","id":"7.15.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.1","id":"7.15.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {% raw -%} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} diff --git a/salt/pcap/files/config b/salt/pcap/files/config index 900234bc1..24f9a579e 100644 --- a/salt/pcap/files/config +++ b/salt/pcap/files/config @@ -1,20 +1,23 @@ -{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} -{%- set diskfreepercentage = salt['pillar.get']('steno:diskfreepercentage', 10) %} -{%- set maxfiles = salt['pillar.get']('steno:maxfiles', 30000) %} - - +{%- set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %} +{%- set DISKFREEPERCENTAGE = salt['pillar.get']('steno:diskfreepercentage', 10) %} +{%- set MAXFILES = salt['pillar.get']('steno:maxfiles', 30000) %} +{%- set BLOCKS = salt['pillar.get']('steno:blocks', 2048) %} +{%- set FILEMB = salt['pillar.get']('steno:filemb', 4096) %} +{%- set AIOPS = salt['pillar.get']('steno:aiops', 128) %} +{%- set THREADS = salt['pillar.get']('steno:threads', 1) %} { "Threads": [ - { "PacketsDirectory": "/nsm/pcap" - , "IndexDirectory": "/nsm/pcapindex" - , "MaxDirectoryFiles": {{ maxfiles }} - , "DiskFreePercentage": {{ diskfreepercentage }} - } + { "PacketsDirectory": "/nsm/pcap", "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} } + {%- if THREADS > 1 %} + {%- for i in range(2,THREADS+1) %} + , { "PacketsDirectory": "/nsm/pcap" , "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} } + {%- endfor %} + {%- endif %} ] , "StenotypePath": "/usr/bin/stenotype" - , "Interface": "{{ interface }}" + , "Interface": "{{ INTERFACE }}" , "Port": 1234 , "Host": "127.0.0.1" - , "Flags": ["-v", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] + , "Flags": ["-v", "--blocks={{ BLOCKS }}", "--preallocate_file_mb={{ FILEMB }}", "--aiops={{ AIOPS }}", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] , "CertPath": "/etc/stenographer/certs" } diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index e33ea406b..02128fd3c 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -26,7 +26,7 @@ {%- set ES_USER = '' %} {%- set ES_PASS = '' %} {%- endif %} - +{%- set ES_INDEX_PATTERNS = salt['pillar.get']('soc:es_index_patterns', '*:so-*') %} { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", "server": { @@ -57,6 +57,7 @@ {%- endif %} "username": "{{ ES_USER }}", "password": "{{ ES_PASS }}", + "index": "{{ ES_INDEX_PATTERNS }}", "cacheMs": {{ ES_FIELDCAPS_CACHE }}, "verifyCert": false, "timeoutMs": {{ API_TIMEOUT }} diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index b04b1549d..59d806fe0 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -673,9 +673,22 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false +{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.logstash]] + url = "http://localhost:9600" + collect = ["pipelines"] + {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" + {% endif %} +{%- endif %} + +{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.redis]] + servers = ["tcp://localhost:6379"] +{%- endif %} # # Read metrics from one or more commands that can output to stdout - [[inputs.exec]] commands = [ "/scripts/sostatus.sh" diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index 028637e16..09b7ddb70 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -19,25 +19,30 @@ THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then - TSFILE=/var/log/telegraf/laststenodrop.log - if [ -f "$TSFILE" ]; then - LASTTS=$(cat $TSFILE) - else - LASTTS=0 + CHECKIT=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2|wc -l) + STENOGREP=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2) + + declare RESULT=($STENOGREP) + + CURRENT_PACKETS=$(echo ${RESULT[9]} | awk -F'=' '{print $2 }') + CURRENT_DROPS=$(echo ${RESULT[12]} | awk -F'=' '{print $2 }') + PREVIOUS_PACKETS=$(echo ${RESULT[23]} | awk -F'=' '{print $2 }') + PREVIOUS_DROPS=$(echo ${RESULT[26]} | awk -F'=' '{print $2 }') + + DROPPED=$((CURRENT_DROPS - PREVIOUS_DROPS)) + TOTAL_CURRENT=$((CURRENT_PACKETS + CURRENT_DROPS)) + TOTAL_PAST=$((PREVIOUS_PACKETS + PREVIOUS_DROPS)) + TOTAL=$((TOTAL_CURRENT - TOTAL_PAST)) + + if [ $CHECKIT == 2 ]; then + if [ $DROPPED == 0 ]; then + echo "stenodrop drop=$DROPPED" + else + LOSS=$(echo "4 k $DROPPED $TOTAL / 100 * p" | dc) + echo "stenodrop drop=$LOSS" + fi fi - - # Get the data - LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) - CURRENTTS=$(echo $LOGLINE | awk '{print $1}') - - if [[ "$CURRENTTS" != "$LASTTS" ]]; then - DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') - echo $CURRENTTS > $TSFILE - else - DROP=0 - fi - - echo "stenodrop drop=$DROP" + else exit 0 fi \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index af66896c6..f4d08e9a9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1645,6 +1645,7 @@ manager_pillar() { printf '%s\n'\ "soc:"\ " endgamehost: '$ENDGAMEHOST'"\ + " es_index_patterns: '*:so-*,*:endgame-*'"\ "" >> "$pillar_file" fi }