diff --git a/salt/common/files/daemon.json b/salt/common/files/daemon.json index bc047bc80..c2df49f34 100644 --- a/salt/common/files/daemon.json +++ b/salt/common/files/daemon.json @@ -1,5 +1,5 @@ -{%- set DOCKERRANGE = salt['pillar.get']('docker:range') %} -{%- set DOCKERBIND = salt['pillar.get']('docker:bip') %} +{%- set DOCKERRANGE = salt['pillar.get']('docker:range', '172.17.0.0/24') %} +{%- set DOCKERBIND = salt['pillar.get']('docker:bip', '172.17.0.1/24') %} { "registry-mirrors": [ "https://:5000" ], "bip": "{{ DOCKERBIND }}", @@ -9,4 +9,4 @@ "size" : 24 } ] -} \ No newline at end of file +} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 767f9d21c..ef53ce60f 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -84,11 +84,13 @@ container_list() { TRUSTED_CONTAINERS=( "so-filebeat" "so-idstools" + "so-elasticsearch" "so-logstash" "so-nginx" "so-redis" "so-steno" "so-suricata" + "so-soc" "so-telegraf" "so-zeek" ) diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap index 2dc5b0504..72c199231 100755 --- a/salt/common/tools/sbin/so-import-pcap +++ b/salt/common/tools/sbin/so-import-pcap @@ -217,6 +217,6 @@ https://{{ URLBASE }}/#/hunt?q=import.id:${HASH}%20%7C%20groupby%20event.module% or you can manually set your Time Range to be (in UTC): From: $START_OLDEST To: $END_NEWEST -Please note that it may take 30 seconds or more for events to appear in Onion Hunt. +Please note that it may take 30 seconds or more for events to appear in Hunt. EOF fi diff --git a/salt/common/tools/sbin/so-suricata-testrule b/salt/common/tools/sbin/so-suricata-testrule index 645a0368b..fee70cff5 100644 --- a/salt/common/tools/sbin/so-suricata-testrule +++ b/salt/common/tools/sbin/so-suricata-testrule @@ -29,14 +29,14 @@ echo "Running all.rules and $TESTRULE against the following pcap: $TESTPCAP" echo "" sleep 3 -cp /opt/so/conf/suricata/rules/all.rules /tmp/nids-testing/rules/all.rules -cat $TESTRULE >> /tmp/nids-testing/rules/all.rules rm -rf /tmp/nids-testing/output mkdir -p /tmp/nids-testing/output chown suricata:socore /tmp/nids-testing/output mkdir -p /tmp/nids-testing/rules +cp /opt/so/conf/suricata/rules/all.rules /tmp/nids-testing/rules/all.rules +cat $TESTRULE >> /tmp/nids-testing/rules/all.rules echo "==== Begin Suricata Output ===" diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f9ac6de2b..4f8a00a82 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -158,6 +158,7 @@ copy_new_files() { generate_and_clean_tarballs() { local new_version new_version=$(cat $UPDATE_DIR/VERSION) + [ -d /opt/so/repo ] || mkdir -p /opt/so/repo tar -cxf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR" find "/opt/so/repo" -type f -not -name "$new_version.tar.gz" -exec rm -rf {} \; } @@ -290,7 +291,7 @@ rc3_to_2.3.0() { INSTALLEDVERSION=2.3.0 } -2.3.0_to_2.3.20(){ +2.3.0_to_2.3.20(){DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 # Remove PCAP from global sed '/pcap:/d' /opt/so/saltstack/local/pillar/global.sls sed '/sensor_checkin_interval_ms:/d' /opt/so/saltstack/local/pillar/global.sls @@ -340,6 +341,29 @@ space_check() { } +thehive_maint() { + echo -n "Waiting for TheHive..." + COUNT=0 + THEHIVE_CONNECTED="no" + while [[ "$COUNT" -le 240 ]]; do + curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert" + if [ $? -eq 0 ]; then + THEHIVE_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + if [ "$THEHIVE_CONNECTED" == "yes" ]; then + echo "Migrating thehive databases if needed." + curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" + curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" + fi +} + unmount_update() { cd /tmp umount /tmp/soagupdate @@ -583,9 +607,6 @@ if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then echo "" fi -echo "" -echo "Applying common state for any package updates." -salt-call -l info state.apply common queue=True echo "" echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." salt-call state.highstate -l info queue=True @@ -605,6 +626,7 @@ echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True playbook unmount_update +thehive_maint if [ "$UPGRADESALT" == "1" ]; then echo "" diff --git a/salt/elasticsearch/files/scripts/so-catrust b/salt/elasticsearch/files/scripts/so-catrust index aee83a379..d49a29ce4 100644 --- a/salt/elasticsearch/files/scripts/so-catrust +++ b/salt/elasticsearch/files/scripts/so-catrust @@ -18,6 +18,10 @@ {%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {%- set MANAGER = salt['grains.get']('master') %} . /usr/sbin/so-common + +# Exit on errors, since all lines must succeed +set -e + # Check to see if we have extracted the ca cert. if [ ! -f /opt/so/saltstack/local/salt/common/cacerts ]; then docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} -keystore /etc/pki/ca-trust/extracted/java/cacerts -alias SOSCA -import -file /etc/pki/ca.crt -storepass changeit -noprompt diff --git a/salt/grafana/dashboards/eval/eval.json b/salt/grafana/dashboards/eval/eval.json index ee94504d1..cc4298bb2 100644 --- a/salt/grafana/dashboards/eval/eval.json +++ b/salt/grafana/dashboards/eval/eval.json @@ -16,7 +16,7 @@ "editable": true, "gnetId": 2381, "graphTooltip": 0, - "iteration": 1602101784759, + "id": 3, "links": [], "panels": [ { @@ -24,36 +24,18 @@ "fieldConfig": { "defaults": { "custom": {}, - "unit": "percent", - "min": 0, - "max": 100, + "decimals": 2, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", + "color": "rgb(255, 255, 255)", "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 } ] }, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected" + "unit": "s" }, "overrides": [] }, @@ -63,15 +45,24 @@ "x": 0, "y": 0 }, - "id": 2, - "links": [], + "id": 39, "options": { - "alertThreshold": true + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, "pluginVersion": "7.3.4", "targets": [ { - "dsType": "influxdb", "groupBy": [ { "params": [ @@ -86,7 +77,7 @@ "type": "fill" } ], - "measurement": "cpu", + "measurement": "system", "orderByTime": "ASC", "policy": "default", "refId": "A", @@ -95,19 +86,13 @@ [ { "params": [ - "usage_idle" + "uptime" ], "type": "field" }, { "params": [], - "type": "mean" - }, - { - "params": [ - "* -1 + 100" - ], - "type": "math" + "type": "last" } ] ], @@ -116,94 +101,26 @@ "key": "host", "operator": "=", "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" } - ], - "alias": "Usage" + ] } ], - "title": "{{ SERVERNAME }} - CPU", - "type": "graph", - "cacheTimeout": null, - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "percent", - "$$hashKey": "object:395" - }, - { - "label": null, - "show": false, - "logBase": 1, - "min": null, - "max": null, - "format": "short", - "$$hashKey": "object:396" - } - ], - "xaxis": { - "show": true, - "mode": "time", - "name": null, - "values": [], - "buckets": null - }, - "yaxis": { - "align": false, - "alignLevel": null - }, - "lines": true, - "fill": 1, - "fillGradient": 0, - "linewidth": 1, - "dashes": false, - "hiddenSeries": false, - "dashLength": 10, - "spaceLength": 10, - "points": false, - "pointradius": 2, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": false, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "individual", - "shared": true, - "sort": 0 - }, "timeFrom": null, "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [], - "timeRegions": [] + "title": "{{ SERVERNAME }} - System Uptime", + "type": "stat" }, { + "aliasColors": {}, + "bars": false, "cacheTimeout": null, + "dashLength": 10, + "dashes": false, "datasource": "InfluxDB", "fieldConfig": { "defaults": { "custom": {}, + "links": [], "mappings": [ { "id": 0, @@ -237,27 +154,41 @@ }, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 0 }, + "hiddenSeries": false, "id": 2, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false }, - "pluginVersion": "7.0.5", + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "dsType": "influxdb", @@ -315,8 +246,46 @@ ] } ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, "title": "{{ SERVERNAME }} - CPU", - "type": "gauge" + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "Usage", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -326,7 +295,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -353,9 +323,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": true, "renderer": "flot", @@ -427,7 +398,6 @@ }, "yaxes": [ { - "$$hashKey": "object:198", "decimals": 1, "format": "percent", "label": "", @@ -437,7 +407,6 @@ "show": true }, { - "$$hashKey": "object:199", "format": "short", "label": null, "logBase": 1, @@ -460,7 +429,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -488,10 +458,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pluginVersion": "6.6.2", + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -601,7 +571,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -629,9 +600,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -740,7 +712,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -768,10 +741,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pluginVersion": "6.6.2", + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -887,13 +860,13 @@ "y": 5 }, "hiddenSeries": false, - "id": 73, + "id": 75, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -914,7 +887,7 @@ "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Usage", "groupBy": [ { "params": [ @@ -983,7 +956,7 @@ }, "yaxes": [ { - "$$hashKey": "object:708", + "decimals": 2, "format": "percent", "label": null, "logBase": 1, @@ -992,7 +965,6 @@ "show": true }, { - "$$hashKey": "object:709", "format": "short", "label": null, "logBase": 1, @@ -1027,7 +999,7 @@ "y": 5 }, "hiddenSeries": false, - "id": 74, + "id": 77, "legend": { "avg": false, "current": false, @@ -1054,7 +1026,7 @@ "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Usage", "groupBy": [ { "params": [ @@ -1072,6 +1044,7 @@ "measurement": "disk", "orderByTime": "ASC", "policy": "default", + "queryType": "randomWalk", "refId": "A", "resultFormat": "time_series", "select": [ @@ -1123,7 +1096,7 @@ }, "yaxes": [ { - "$$hashKey": "object:708", + "decimals": 2, "format": "percent", "label": null, "logBase": 1, @@ -1132,7 +1105,6 @@ "show": true }, { - "$$hashKey": "object:709", "format": "short", "label": null, "logBase": 1, @@ -1155,7 +1127,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -1183,10 +1156,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pluginVersion": "6.6.2", + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1301,7 +1274,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -1329,9 +1303,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1446,7 +1421,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -1474,9 +1450,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1591,7 +1568,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -1619,9 +1597,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1746,7 +1725,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -1776,9 +1756,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -2138,7 +2119,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -2165,9 +2147,10 @@ "linewidth": 1, "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2245,7 +2228,6 @@ }, "yaxes": [ { - "$$hashKey": "object:147", "decimals": 1, "format": "decbytes", "label": "", @@ -2255,7 +2237,6 @@ "show": true }, { - "$$hashKey": "object:148", "format": "short", "label": null, "logBase": 1, @@ -2278,7 +2259,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -2306,10 +2288,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pluginVersion": "6.6.2", + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2418,7 +2400,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -2446,9 +2429,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2557,7 +2541,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -2585,9 +2570,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2822,7 +2808,8 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -2850,9 +2837,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2986,9 +2974,10 @@ ], "fields": "", "values": false - } + }, + "textMode": "auto" }, - "pluginVersion": "7.0.5", + "pluginVersion": "7.3.4", "targets": [ { "groupBy": [ @@ -3038,18 +3027,25 @@ "title": "Zeek Restarts via Healthcheck", "type": "stat" }, - - - { + "cacheTimeout": null, "datasource": "InfluxDB", "fieldConfig": { "defaults": { "custom": {}, - "unit": "s", - "min": 0, - "max": null, "decimals": 2, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 1209600, + "min": 0, + "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ @@ -3067,16 +3063,7 @@ } ] }, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected" + "unit": "s" }, "overrides": [] }, @@ -3089,7 +3076,16 @@ "id": 22, "links": [], "options": { - "alertThreshold": true + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true }, "pluginVersion": "7.3.4", "targets": [ @@ -3134,81 +3130,13 @@ "operator": "=", "value": "{{ SERVERNAME }}" } - ], - "alias": "Oldest Pcap" + ] } ], - "title": "{{ SERVERNAME }} - PCAP Retention", - "type": "graph", - "renderer": "flot", - "yaxes": [ - { - "label": "", - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "s", - "$$hashKey": "object:643", - "decimals": 2 - }, - { - "label": null, - "show": false, - "logBase": 1, - "min": null, - "max": null, - "format": "short", - "$$hashKey": "object:644" - } - ], - "xaxis": { - "show": true, - "mode": "time", - "name": null, - "values": [], - "buckets": null - }, - "yaxis": { - "align": false, - "alignLevel": null - }, - "lines": true, - "fill": 1, - "linewidth": 1, - "dashLength": 10, - "spaceLength": 10, - "pointradius": 2, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "tooltip": { - "value_type": "individual", - "shared": true, - "sort": 0 - }, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [], - "timeRegions": [], - "cacheTimeout": null, "timeFrom": null, "timeShift": null, - "fillGradient": 0, - "dashes": false, - "hiddenSeries": false, - "points": false, - "bars": false, - "stack": false, - "percentage": false, - "steppedLine": false + "title": "{{ SERVERNAME }} - PCAP Retention", + "type": "gauge" }, { "aliasColors": { @@ -3225,7 +3153,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -3255,9 +3184,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3505,7 +3435,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -3537,9 +3468,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3777,7 +3709,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -3807,9 +3740,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3838,7 +3772,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3938,7 +3872,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -3968,9 +3903,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4162,7 +4098,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -4191,9 +4128,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4355,7 +4293,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -4385,9 +4324,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4416,7 +4356,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -4480,7 +4420,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -4577,7 +4517,8 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "custom": {}, + "links": [] }, "overrides": [] }, @@ -4607,9 +4548,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4706,91 +4648,11 @@ } ], "refresh": "30s", - "schemaVersion": 25, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { - "list": [ - { - "auto": true, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "text": "10s", - "value": "10s" - }, - "hide": 0, - "label": null, - "name": "Interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_Interval" - }, - { - "selected": true, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] + "list": [] }, "time": { "from": "now-1h", @@ -4798,7 +4660,6 @@ }, "timepicker": { "refresh_intervals": [ - "5s", "10s", "30s", "1m", @@ -4825,4 +4686,4 @@ "title": "Evaluation Mode - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", "version": 1 -} +} \ No newline at end of file diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index e98bbecf5..b8580fd86 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -133,18 +133,19 @@ append_so-steno_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-steno - - unless: grep so-steno /opt/so/conf/so-status/so-status.conf + - unless: grep -q so-steno /opt/so/conf/so-status/so-status.conf - {% if STENOOPTIONS.status == 'running' %} -delete_so-steno_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-steno$ - {% elif STENOOPTIONS.status == 'stopped' %} + + {% if not STENOOPTIONS.start %} so-steno_so-status.disabled: file.comment: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-steno$ + {% else %} +delete_so-steno_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-steno$ {% endif %} {% else %} diff --git a/salt/playbook/automation_user_create.sls b/salt/playbook/automation_user_create.sls index 51eae1796..e333a4a99 100644 --- a/salt/playbook/automation_user_create.sls +++ b/salt/playbook/automation_user_create.sls @@ -8,12 +8,12 @@ include: wait_for_playbook: cmd.run: - name: until nc -z {{ MAINIP }} 3200; do sleep 1; done - - timeout: 30 - - onchanges: - - cmd: create_user + - timeout: 300 create_user: cmd.script: - source: salt://playbook/files/automation_user_create.sh - cwd: /root - template: jinja + - onchanges: + - cmd: wait_for_playbook diff --git a/salt/playbook/files/automation_user_create.sh b/salt/playbook/files/automation_user_create.sh index 03736e521..86f279378 100644 --- a/salt/playbook/files/automation_user_create.sh +++ b/salt/playbook/files/automation_user_create.sh @@ -2,6 +2,8 @@ # {%- set admin_pass = salt['pillar.get']('secrets:playbook_admin', None) -%} # {%- set automation_pass = salt['pillar.get']('secrets:playbook_automation', None) %} +set -e + local_salt_dir=/opt/so/saltstack/local try_count=6 @@ -44,7 +46,11 @@ while [[ $try_count -le 6 ]]; do echo " api_key: ${automation_api_key}" } >> $local_salt_dir/pillar/global.sls fi + exit 0 fi ((try_count++)) sleep "${interval}s" done + +# Timeout exceeded, exit with non-zero exit code +exit 1 diff --git a/salt/soc/files/soc/changes.json b/salt/soc/files/soc/changes.json index 90f71f940..2736e73b8 100644 --- a/salt/soc/files/soc/changes.json +++ b/salt/soc/files/soc/changes.json @@ -1,42 +1,54 @@ { - "title": "Security Onion 2.3.10 is here!", + "title": "Security Onion 2.3.20 is here!", "changes": [ - { "summary": "UEFI installs with multiple disks should work as intended now." }, - { "summary": "Telegraf scripts will now make sure they are not already running before execution." }, - { "summary": "You are now prompted during setup if you want to change the docker IP range. If you change this it needs to be the same on all nodes in the grid." }, - { "summary": "Soup will now download the new containers before stopping anything. If anything fails it will now exit and leave the grid at the current version." }, - { "summary": "All containers are now hosted on quay.io to prevent pull limitations. We are now using GPG keys to determine if the image is from Security Onion." }, - { "summary": "Osquery installers have been updated to osquery 4.5.1." }, - { "summary": "Fix for bug where Playbook was not removing the Elastalert rules for inactive Plays." }, - { "summary": "Exifdata reported by Strelka is now constrained to a single multi-valued field to prevent mapping explosion (scan.exiftool)." }, - { "summary": "Resolved issue with Navigator layer(s) not loading correctly." }, - { "summary": "Wazuh authd is now started by default on port 1515/tcp." }, - { "summary": "Wazuh API default credentials are now removed after setup. Scripts have been added for API user management." }, - { "summary": "Upgraded Salt to 3002.2 due to CVEs." }, - { "summary": "If salt-minion is unable to apply states after the defined threshold, we assume salt-minion is in a bad state and the salt-minion service will be restarted." }, - { "summary": "Fixed bug that prevented mysql from installing for Fleet if Playbook wasn't also installed." }, - { "summary": "so-status will now show STARTING or WAIT_START, instead of ERROR, if so-status is run before a salt highstate has started or finished for the first time after system startup" }, - { "summary": "Stenographer can now be disabled on a sensor node by setting the pillar steno:enabled:false in it's minion.sls file or globally if set in the global.sls file" }, - { "summary": "Added so-ssh-harden script that runs the commands listed in https://docs.securityonion.net/en/2.3/ssh.html" }, - { "summary": "NGINX now redirects the browser to the hostname/IP address/FQDN based on global:url_base" }, - { "summary": "MySQL state now waits for MySQL server to respond to a query before completeing" }, - { "summary": "Added Analyst option to network installs" }, - { "summary": "Acknowledging (and Escalating) alerts did not consistently remove the alert from the visible list; this has been corrected." }, - { "summary": "Escalating alerts that have a rule.case_template field defined will automatically assign that case template to the case generated in TheHive." }, - { "summary": "Alerts and Hunt interface quick action bar has been converted into a vertical menu to improve quick action option clarity. Related changes also eliminated the issues that occurred when the quick action bar was appearing to the left of the visible browser area." }, - { "summary": "Updated Go to newer version to fix a timezone, daylight savings time (DST) issue that resulted in Alerts and Hunt interfaces not consistently showing results." }, - { "summary": "Improved Hunt and Alert table sorting." }, - { "summary": "Alerts interface now allows absolute time searches." }, - { "summary": "Alerts interface 'Hunt' quick action is now working as intended." }, - { "summary": "Alerts interface 'Ack' icon tooltip has been changed from 'Dismiss' to 'Acknowledge' for consistency." }, - { "summary": "Hunt interface bar charts will now show the quick action menu when clicked instead of assuming the click was intended to add an include filter." }, - { "summary": "Hunt interface quick action will now cast a wider net on field searches." }, - { "summary": "Now explicitly preventing the use of a dollar sign ($) character in web user passwords during setup." }, - { "summary": "Cortex container will now restart properly if the SO host was not gracefully shutdown." }, - { "summary": "Added syslog plugin to the logstash container; this is not in-use by default but available for those users that choose to use it." }, - { "summary": "Winlogbeat download package is now available from the SOC Downloads interface." }, - { "summary": "Upgraded Kratos authentication system." }, - { "summary": "Added new Reset Defaults button to the SOC Profile Settings interface which allows users to reset all local browser SOC customizations back to their defaults. This includes things like default sort column, sort order, items per page, etc." }, - { "summary": "Known Issues " } + { "summary": "soup has been refactored. You will need to run it a few times to get all the changes properly. We are working on making this even easier for future releases."}, + { "summary": "soup now has awareness of Elastic Features and now downloads the appropriate Docker containers."}, + { "summary": "The Sensors interface has been renamed to Grid. This interface now includes all Security Onion nodes."}, + { "summary": "Grid interface now includes the status of the node. The status currently shows either Online (blue) or Offline (orange). If a node does not check-in on time then it will be marked as Offline."}, + { "summary": "Grid interface now includes the IP and Role of each node in the grid."}, + { "summary": "Grid interface includes a new Filter search input to filter the visible list of grid nodes to a desired subset. As an example, typing in “sensor” will hide all nodes except those that behave as a sensor."}, + { "summary": "The Grid description field can now be customized via the local minion pillar file for each node."}, + { "summary": "SOC will now draw attention to an unhealthy situation within the grid or with the connection between the user’s browser and the manager node. For example, when the Grid has at least one Offline node the SOC interface will show an exclamation mark in front of the browser tab’s title and an exclamation mark next to the Grid menu option in SOC. Additionally, the favicon will show an orange marker in the top-right corner (dynamic favicons not supported in Safari). Additionally, if the user’s web browser is unable to communicate with the manager the unhealth indicators appear along with a message at the top of SOC that states there is a connection problem."}, + { "summary": "Docker has been upgraded to the latest version."}, + { "summary": "Docker should be more reliable now as Salt is now managing daemon.json."}, + { "summary": "You can now install Elastic in a traditional cluster. When setting up the manager select Advanced and follow the prompts. Replicas are controlled in global.sls."}, + { "summary": "You can now use Hot and Warm routing with Elastic in a traditional cluster. You can change the box.type in the minion’s sls file. You will need to create a curator job to re-tag the indexes based on your criteria."}, + { "summary": "Telegraf has been updated to version 1.16.3."}, + { "summary": "Grafana has been updated to 7.3.4 to resolve some XSS vulnerabilities."}, + { "summary": "Grafana graphs have been changed to graphs vs guages so alerting can be set up."}, + { "summary": "Grafana is now completely pillarized, allowing users to customize alerts and making it customizable for email, Slack, etc. See the docs here."}, + { "summary": "Yara rules now should properly install on non-airgap installs. Previously, users had to wait for an automated job to place them in the correct location."}, + { "summary": "Strelka backend will not stop itself any more. Previously, its behavior was to shut itself down after fifteen minutes and wait for Salt to restart it to look for work before shutting down again."}, + { "summary": "Strelka daily rule updates are now logged to /nsm/strelka/log/yara-update.log"}, + { "summary": "Several changes to the setup script to improve install reliability."}, + { "summary": "Airgap now supports the import node type."}, + { "summary": "Custom Zeek file extraction values in the pillar now work properly."}, + { "summary": "TheHive has been updated to support Elastic 7."}, + { "summary": "Cortex image now includes whois package to correct an issue with the CERTatPassiveDNS analyzer."}, + { "summary": "Hunt and Alert quick action menu has been refactored into submenus."}, + { "summary": "New clipboard quick actions now allow for copying fields or entire events to the clipboard."}, + { "summary": "PCAP Add Job form now retains previous job details for quickly adding additional jobs. A new Clear button now exists at the bottom of this form to clear out these fields and forget the previous job details."}, + { "summary": "PCAP Add Job form now allows users to perform arbitrary PCAP lookups of imported PCAP data (data imported via the so-import-pcap script)."}, + { "summary": "Downloads page now allows direct download of Wazuh agents for Linux, Mac, and Windows from the manager, and shows the version of Wazuh and Elastic installed with Security Onion."}, + { "summary": "PCAP job interface now shows additional job filter criteria when expanding the job filter details."}, + { "summary": "Upgraded authentication backend to Kratos 0.5.5."}, + { "summary": "SOC tables with the “Rows per Page” dropdown no longer show truncated page counts."}, + { "summary": "Several Hunt errors are now more descriptive, particularly those around malformed queries."}, + { "summary": "SOC Error banner has been improved to avoid showing raw HTML syntax, making connection and server-side errors more readable."}, + { "summary": "Hunt and Alerts interfaces will now allow pivoting to PCAP from a group of results if the grouped results contain a network.community_id field."}, + { "summary": "New “Correlate” quick action will pivot to a new Hunt search for all events that can be correlated by at least one of various event IDs."}, + { "summary": "Fixed bug that caused some Hunt queries to not group correctly without a .keyword suffix. This has been corrected so that the .keyword suffix is no longer necessary on those groupby terms."}, + { "summary": "Fixed issue where PCAP interface loses formatting and color coding when opening multiple PCAP tabs."}, + { "summary": "Alerts interface now has a Refresh button that allows users to refresh the current alerts view without refreshing the entire SOC application."}, + { "summary": "Hunt and Alerts interfaces now have an auto-refresh dropdown that will automatically refresh the current view at the selected frequency."}, + { "summary": "The so-elastalert-test script has been refactored to work with Security Onion 2.3."}, + { "summary": "The included Logstash image now includes Kafka plugins."}, + { "summary": "Wazuh agent registration process has been improved to support slower hardware and networks."}, + { "summary": "An Elasticsearch ingest pipeline has been added for suricata.ftp_data."}, + { "summary": "Elasticsearch’s indices.query.bool.max_clause_count value has been increased to accommodate a slightly larger number of fields (1024 -> 1500) when querying using a wildcard."}, + { "summary": "On nodes being added to an existing grid, setup will compare the version currently being installed to the manager (>=2.3.20), pull the correct Security Onion version from the manager if there is a mismatch, and run that version."}, + { "summary": "Setup will gather any errors found during a failed install into /root/errors.log for easy copy/paste and debugging."}, + { "summary": "Selecting Suricata as the metadata engine no longer results in the install failing."}, + { "summary": "so-rule-update now accepts arguments to idstools. For example, so-rule-update -f will force idstools to pull rules, ignoring the default 15-minute pull limit."} ] } diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 0c50bb5d1..99609be32 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -167,6 +167,14 @@ append_so-suricata_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-suricata + - unless: grep -q so-suricata /opt/so/conf/so-status/so-status.conf + +{% if grains.role == 'so-import' %} +disable_so-suricata_so-status.conf: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-suricata$ +{% endif %} surilogrotate: file.managed: diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index 8d400ca1e..1ff34ceae 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -48,6 +48,7 @@ so-telegraf: - HOST_ETC=/host/etc - HOST_SYS=/host/sys - HOST_MOUNT_PREFIX=/host + - GODEBUG=x509ignoreCN=0 - network_mode: host - binds: - /opt/so/log/telegraf:/var/log/telegraf:rw @@ -84,4 +85,4 @@ telegraf_state_not_allowed: test.fail_without_changes: - name: telegraf_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index f6edae136..6fa289d5c 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -200,6 +200,14 @@ append_so-zeek_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-zeek + - unless: grep -q so-zeek /opt/so/conf/so-status/so-status.conf + +{% if grains.role == 'so-import' %} +disable_so-zeek_so-status.conf: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-zeek$ +{% endif %} {% else %} diff --git a/screenshots/alerts-1.png b/screenshots/alerts-1.png index 140150c77..099710f4f 100644 Binary files a/screenshots/alerts-1.png and b/screenshots/alerts-1.png differ diff --git a/screenshots/hunt-1.png b/screenshots/hunt-1.png index aa7ae7c1e..089713847 100644 Binary files a/screenshots/hunt-1.png and b/screenshots/hunt-1.png differ diff --git a/setup/so-functions b/setup/so-functions index 5f98e685e..78bde3a95 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -719,7 +719,7 @@ create_local_directories() { for d in $(find $PILLARSALTDIR/$i -type d); do suffixdir=${d//$PILLARSALTDIR/} if [ ! -d "$local_salt_dir/$suffixdir" ]; then - mkdir -v "$local_salt_dir$suffixdir" >> "$setup_log" 2>&1 + mkdir -pv "$local_salt_dir$suffixdir" >> "$setup_log" 2>&1 fi done chown -R socore:socore "$local_salt_dir/$i" @@ -1198,7 +1198,10 @@ manager_global() { fi if [ -z "$DOCKERNET" ]; then - DOCKERNET=172.17.0.0 + DOCKERNET=172.17.0.0 + DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 + else + DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 fi # Create a global file for global values @@ -1276,9 +1279,9 @@ manager_global() { " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\ - " cluster_routing_allocation_disk_watermark_low: '95%'"\ - " cluster_routing_allocation_disk_watermark_high: '98%'"\ - " cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\ + " cluster_routing_allocation_disk_watermark_low: '95%'"\ + " cluster_routing_allocation_disk_watermark_high: '98%'"\ + " cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\ " index_settings:"\ " so-beats:"\ " shards: 1"\ @@ -1346,6 +1349,9 @@ manager_global() { " playbook:"\ " rulesets:"\ " - windows"\ + "docker:"\ + " range: '$DOCKERNET/24'"\ + " bip: '$DOCKERBIP'"\ "redis_settings:"\ " redis_maxmemory: 812" >> "$global_pillar" @@ -1525,7 +1531,7 @@ reinstall_init() { if command -v docker &> /dev/null; then # Stop and remove all so-* containers so files can be changed with more safety - if [ $(docker ps -a -q --filter "name=so-") -gt 0 ]; then + if [ $(docker ps -a -q --filter "name=so-" | wc -l) -gt 0 ]; then docker stop $(docker ps -a -q --filter "name=so-") docker rm -f $(docker ps -a -q --filter "name=so-") fi @@ -1941,7 +1947,6 @@ sensor_pillar() { if [ "$HNSENSOR" != 'inherit' ]; then echo " hnsensor: $HNSENSOR" >> "$pillar_file" fi - } set_default_log_size() { diff --git a/setup/so-whiptail b/setup/so-whiptail index af7f6da2f..d60352bdd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -836,7 +836,7 @@ whiptail_manager_adv_escluster(){ [ -n "$TESTING" ] && return whiptail --title "Security Onion Setup" --yesno \ - "Do you want to set up a traditional ES cluster?" 8 75 + "Do you want to set up a traditional ES cluster for using replicas and/or Hot-Warm indices? Recommended only for those who have experience with ES clustering! " 12 75 local exitstatus=$?