Merge branch 'dev' into foxtrot

salt/common/tools/sbin/so-elastalert-test

@@ -70,7 +70,7 @@ do
 done
 
 docker_exec(){
-	CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/config/elastalert_config.yaml $OPTIONS"
+	CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/elastalert/config.yaml $OPTIONS"
 	if [ "${RESULTS_TO_LOG,,}" = "y" ] ; then
 		$CMD > "$FILE_SAVE_LOCATION"
 	else

salt/common/tools/sbin/so-filebeat-module-setup

@@ -54,7 +54,7 @@ PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_
 if [[ "$PIPELINES" -lt 5 ]]; then
   echo "Setting up ingest pipeline(s)"
 
-  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler
+  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler
   do
     echo "Loading $MODULE"
     docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML

salt/common/tools/sbin/so-user

@@ -182,6 +182,10 @@ function ensureRoleFileExists() {
       echo "Database file does not exist yet, installation is likely not yet complete."
     fi
 
+    if [[ -d "$socRolesFile" ]]; then
+      echo "Removing invalid roles directory created by Docker"
+      rm -fr "$socRolesFile"
+    fi
     mv "${rolesTmpFile}" "${socRolesFile}"
   fi
 }
@@ -445,7 +449,7 @@ function deleteUser() {
 
   rolesTmpFile="${socRolesFile}.tmp"
   createFile "$rolesTmpFile" "$soUID" "$soGID"
-  grep -v "$id" "$socRolesFile" > "$rolesTmpFile"
+  grep -v "$identityId" "$socRolesFile" > "$rolesTmpFile"
   mv "$rolesTmpFile" "$socRolesFile"
 }
 

salt/filebeat/thirdpartydefaults.yaml

@@ -244,6 +244,23 @@ third_party_filebeat:
         var.input: udp
         var.syslog_host: 0.0.0.0
         var.syslog_port: 9501
+    threatintel:
+      abuseurl:
+        enabled: false
+      abusemalware:
+        enabled: false
+      misp:
+        enabled: false
+      malwarebazaar:
+        enabled: false
+      otx:
+        enabled: false
+      anomali:
+        enabled: false
+      anomalithreatstream:
+        enabled: false
+      recordedfuture:
+        enabled: false
     zscaler:
       zia:
         enabled: false 

salt/kibana/files/saved_objects.ndjson.jinja

@@ -462,7 +462,7 @@
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"}
 {% endraw -%}
-{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.0","id":"7.15.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="}
+{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.1","id":"7.15.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="}
 {% raw -%}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"}

salt/pcap/files/config

@@ -1,20 +1,23 @@
-{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
+{%- set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %}
-{%- set diskfreepercentage = salt['pillar.get']('steno:diskfreepercentage', 10) %}
+{%- set DISKFREEPERCENTAGE = salt['pillar.get']('steno:diskfreepercentage', 10) %}
-{%- set maxfiles = salt['pillar.get']('steno:maxfiles', 30000) %}
+{%- set MAXFILES = salt['pillar.get']('steno:maxfiles', 30000) %}
-
+{%- set BLOCKS = salt['pillar.get']('steno:blocks', 2048) %}
-
+{%- set FILEMB = salt['pillar.get']('steno:filemb', 4096) %}
+{%- set AIOPS = salt['pillar.get']('steno:aiops', 128) %} 
+{%- set THREADS = salt['pillar.get']('steno:threads', 1) %}
 {
   "Threads": [
-    { "PacketsDirectory": "/nsm/pcap"
+    { "PacketsDirectory": "/nsm/pcap", "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} }
-    , "IndexDirectory": "/nsm/pcapindex"
+  {%- if THREADS > 1 %}
-    , "MaxDirectoryFiles": {{ maxfiles }}
+      {%- for i in range(2,THREADS+1) %}
-    , "DiskFreePercentage": {{ diskfreepercentage }}
+    , { "PacketsDirectory": "/nsm/pcap" , "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} }
-    }
+      {%- endfor %}
+  {%- endif %}
   ]
   , "StenotypePath": "/usr/bin/stenotype"
-  , "Interface": "{{ interface }}"
+  , "Interface": "{{ INTERFACE }}"
   , "Port": 1234
   , "Host": "127.0.0.1"
-  , "Flags": ["-v", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}]
+  , "Flags": ["-v", "--blocks={{ BLOCKS }}", "--preallocate_file_mb={{ FILEMB }}", "--aiops={{ AIOPS }}", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}]
   , "CertPath": "/etc/stenographer/certs"
 }

salt/soc/files/soc/soc.json

@@ -26,7 +26,7 @@
 {%-   set ES_USER = '' %}
 {%-   set ES_PASS = '' %}
 {%- endif %}
-
+{%- set ES_INDEX_PATTERNS = salt['pillar.get']('soc:es_index_patterns', '*:so-*') %}
 {
   "logFilename": "/opt/sensoroni/logs/sensoroni-server.log",
   "server": {
@@ -57,6 +57,7 @@
         {%- endif %}
         "username": "{{ ES_USER }}",
         "password": "{{ ES_PASS }}",
+	"index": "{{ ES_INDEX_PATTERNS }}",
         "cacheMs": {{ ES_FIELDCAPS_CACHE }},
         "verifyCert": false,
         "timeoutMs": {{ API_TIMEOUT }}

salt/telegraf/etc/telegraf.conf

@@ -673,9 +673,22 @@
 #   ## Use TLS but skip chain & host verification
 #   # insecure_skip_verify = false
 
+{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%}
+[[inputs.logstash]]
+  url = "http://localhost:9600"
+  collect = ["pipelines"]
+  {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
+  username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}"
+  password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
+  {% endif %}
+{%- endif %}
+
+{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%}
+[[inputs.redis]]
+  servers = ["tcp://localhost:6379"]
+{%- endif %}
 
 # # Read metrics from one or more commands that can output to stdout
-
 [[inputs.exec]]
    commands = [
       "/scripts/sostatus.sh"

salt/telegraf/scripts/stenoloss.sh

@@ -19,25 +19,30 @@ THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
 if [ ! "$THEGREP" ]; then
 
-    TSFILE=/var/log/telegraf/laststenodrop.log
+    CHECKIT=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2|wc -l)
-    if [ -f "$TSFILE" ]; then
+    STENOGREP=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2)
-        LASTTS=$(cat $TSFILE)
+
+    declare RESULT=($STENOGREP)
+
+    CURRENT_PACKETS=$(echo ${RESULT[9]} | awk -F'=' '{print $2 }')
+    CURRENT_DROPS=$(echo ${RESULT[12]} | awk -F'=' '{print $2 }')
+    PREVIOUS_PACKETS=$(echo ${RESULT[23]} | awk -F'=' '{print $2 }')
+    PREVIOUS_DROPS=$(echo ${RESULT[26]} | awk -F'=' '{print $2 }')
+
+    DROPPED=$((CURRENT_DROPS - PREVIOUS_DROPS))
+    TOTAL_CURRENT=$((CURRENT_PACKETS + CURRENT_DROPS))
+    TOTAL_PAST=$((PREVIOUS_PACKETS + PREVIOUS_DROPS))
+    TOTAL=$((TOTAL_CURRENT - TOTAL_PAST))
+
+    if [ $CHECKIT == 2 ]; then
+      if [ $DROPPED == 0 ]; then
+        echo "stenodrop drop=$DROPPED"
       else
-        LASTTS=0
+        LOSS=$(echo "4 k $DROPPED $TOTAL / 100 * p" | dc)
+        echo "stenodrop drop=$LOSS"
+      fi
     fi
     
-    # Get the data
-    LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop)
-    CURRENTTS=$(echo $LOGLINE | awk '{print $1}')
-
-    if [[ "$CURRENTTS" != "$LASTTS" ]]; then
-      DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}')
-      echo $CURRENTTS > $TSFILE
-    else
-      DROP=0
-    fi
-
-    echo "stenodrop drop=$DROP"
 else
     exit 0
 fi

setup/so-functions

@@ -1645,6 +1645,7 @@ manager_pillar() {
         	printf '%s\n'\
 			"soc:"\
                 	"  endgamehost: '$ENDGAMEHOST'"\
+			"  es_index_patterns: '*:so-*,*:endgame-*'"\
                 	"" >> "$pillar_file"
 	fi
 }