Merge branch 'dev' into feature/setup-changes

2025-12-16 22:12:48 +01:00 · 2020-07-01 12:56:57 -04:00
parent 90f4b8e043 7bb97f2b2d
commit 19b997ece0
17 changed files with 299 additions and 25 deletions
--- a/salt/common/maps/eval.map.jinja
+++ b/salt/common/maps/eval.map.jinja
@@ -14,6 +14,7 @@
    'so-zeek',
    'so-curator',
    'so-elastalert',
-    'so-soctopus'
+    'so-soctopus',
+    'so-sensoroni'
  ]
 } %}
--- a/salt/common/maps/heavynode.map.jinja
+++ b/salt/common/maps/heavynode.map.jinja
@@ -9,6 +9,7 @@
    'so-steno',
    'so-suricata',
    'so-wazuh',
-    'so-filebeat
+    'so-filebeat',
+    'so-sensoroni'
  ]
 } %}
--- a/salt/common/maps/sensor.map.jinja
+++ b/salt/common/maps/sensor.map.jinja
@@ -3,6 +3,7 @@
    'so-telegraf',
    'so-steno',
    'so-suricata',
-    'so-filebeat'
+    'so-filebeat',
+    'so-sensoroni'
  ]
 } %}
--- a/salt/common/maps/standalone.map.jinja
+++ b/salt/common/maps/standalone.map.jinja
@@ -16,6 +16,7 @@
    'so-suricata',
    'so-steno',
    'so-dockerregistry',
-    'so-soctopus'
+    'so-soctopus',
+    'so-sensoroni'
  ]
 } %}
--- a/salt/common/tools/sbin/so-allow
+++ b/salt/common/tools/sbin/so-allow
@@ -88,6 +88,7 @@ if [ "$SKIP" -eq 0 ]; then
    echo "[w] - Wazuh agent - port 1514/tcp/udp"
    echo "[p] - Wazuh API - port 55000/tcp"
    echo "[r] - Wazuh registration service - 1515/tcp"
+    echo ""
    echo "Please enter your selection (a - analyst, b - beats, o - osquery, w - wazuh):"
    read -r ROLE
    echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):"
--- a/salt/common/tools/sbin/so-import-pcap
+++ b/salt/common/tools/sbin/so-import-pcap
@@ -0,0 +1,221 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+{% set MASTER = salt['grains.get']('master') %}
+{% set VERSION = salt['pillar.get']('static:soversion') %}
+{%- set MASTERIP = salt['pillar.get']('static:masterip') -%}
+
+function usage {
+  cat << EOF
+Usage:
+Please supply at least one pcap file.
+
+For example, to import a single pcap named import.pcap:
+so-import-pcap import.pcap
+
+To import multiple pcaps:
+so-import-pcap import1.pcap import2.pcap
+
+** IMPORTANT **
+Security Onion installations contain processes that automatically discard old data. Therefore, imports of old network traffic might immediately be erased, unless those processes are first disabled. 
+
+EOF
+}
+
+function pcapinfo() {
+  PCAP=$1
+  ARGS=$2
+  docker run --rm -v $PCAP:/input.pcap --entrypoint capinfos {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap $ARGS
+}
+
+function pcapfix() {
+  PCAP=$1
+  PCAP_OUT=$2
+  docker run --rm -v $PCAP:/input.pcap -v $PCAP_OUT:$PCAP_OUT --entrypoint pcapfix {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap -o $PCAP_OUT > /dev/null 2>&1
+}
+
+function suricata() {
+  PCAP=$1
+  HASH=$2
+
+  NSM_PATH=/nsm/import/${HASH}/suricata
+  mkdir -p $NSM_PATH
+  chown suricata:socore $NSM_PATH
+  LOG_PATH=/opt/so/log/suricata/import/${HASH}
+  mkdir -p $LOG_PATH
+  chown suricata:suricata $LOG_PATH
+  docker run --rm \
+    -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
+    -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
+    -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
+    -v ${LOG_PATH}:/var/log/suricata/:rw \
+    -v ${NSM_PATH}/:/nsm/:rw \
+    -v $PCAP:/input.pcap:ro \
+    -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
+    {{ MASTER }}:5000/soshybridhunter/so-suricata:{{ VERSION }} \
+    --runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1
+}
+
+function zeek() {
+  PCAP=$1
+  HASH=$2
+
+  NSM_PATH=/nsm/import/${HASH}/zeek
+  mkdir -p $NSM_PATH/logs
+  mkdir -p $NSM_PATH/spool
+  mkdir -p $NSM_PATH/extracted
+  chown -R zeek:socore $NSM_PATH
+  docker run --rm \
+    -v $NSM_PATH:/nsm:rw \
+    -v $PCAP:/input.pcap:ro \
+    -v /opt/so/conf/zeek/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro \
+    -v /opt/so/conf/zeek/node.cfg:/opt/zeek/etc/node.cfg:ro \
+    -v /opt/so/conf/zeek/zeekctl.cfg:/opt/zeek/etc/zeekctl.cfg:ro \
+    -v /opt/so/conf/zeek/policy/securityonion:/opt/zeek/share/zeek/policy/securityonion:ro \
+    -v /opt/so/conf/zeek/policy/custom:/opt/zeek/share/zeek/policy/custom:ro \
+    -v /opt/so/conf/zeek/policy/cve-2020-0601:/opt/zeek/share/zeek/policy/cve-2020-0601:ro \
+    -v /opt/so/conf/zeek/policy/intel:/opt/zeek/share/zeek/policy/intel:rw \
+    -v /opt/so/conf/zeek/bpf:/opt/zeek/etc/bpf:ro \
+    --entrypoint /opt/zeek/bin/zeek \
+    -w /nsm/zeek/logs \
+    {{ MASTER }}:5000/soshybridhunter/so-zeek:{{ VERSION }} \
+    -C -r /input.pcap local > $NSM_PATH/logs/console.log 2>&1
+}
+
+# if no parameters supplied, display usage
+if [ $# -eq 0 ]; then
+  usage
+  exit
+fi
+
+# ensure this is a manager node
+if [ ! -d /opt/so/conf/soc ]; then
+  echo "This procedure must be run on a manager node."
+  exit
+fi
+
+# verify that all parameters are files
+for i in "$@"; do
+  if ! [ -f "$i" ]; then
+    usage
+    echo "\"$i\" is not a valid file!"
+    exit
+  fi
+done
+
+# track if we have any valid or invalid pcaps
+INVALID_PCAPS="no"
+VALID_PCAPS="no"
+
+# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
+START_OLDEST="2050-12-31"
+END_NEWEST="1971-01-01"
+
+# paths must be quoted in case they include spaces
+for PCAP in "$@"; do
+  PCAP=$(/usr/bin/realpath "$PCAP")
+  echo "Processing Import: ${PCAP}"
+  echo "- verifying file"
+  if ! pcapinfo "${PCAP}" > /dev/null 2>&1; then
+    # try to fix pcap and then process the fixed pcap directly
+    PCAP_FIXED=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap`
+    echo "- attempting to recover corrupted PCAP file"
+    pcapfix "${PCAP}" "${PCAP_FIXED}"
+    PCAP="${PCAP_FIXED}"
+    TEMP_PCAPS+=(${PCAP_FIXED})
+  fi
+
+  # generate a unique hash to assist with dedupe checks
+  HASH=$(md5sum "${PCAP}" | awk '{ print $1 }')
+  HASH_DIR=/nsm/import/${HASH}
+  echo "- assigning unique identifier to import: $HASH"
+
+  if [ -d $HASH_DIR ]; then
+    echo "- this PCAP has already been imported; skipping"
+    INVALID_PCAPS="yes"
+  elif pcapinfo "${PCAP}" |egrep -q "Last packet time:    1970-01-01|Last packet time:    n/a"; then
+    echo "- this PCAP file is invalid; skipping"
+    INVALID_PCAPS="yes"
+  else
+    VALID_PCAPS="yes"
+
+    PCAP_DIR=$HASH_DIR/pcap
+    mkdir -p $PCAP_DIR
+
+    # generate IDS alerts and write them to standard pipeline
+    echo "- analyzing traffic with Suricata"
+    suricata "${PCAP}" $HASH
+
+    # generate Zeek logs and write them to a unique subdirectory in /nsm/import/bro/
+    # since each run writes to a unique subdirectory, there is no need for a lock file
+    echo "- analyzing traffic with Zeek"
+    zeek "${PCAP}" $HASH
+
+    START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}')
+    END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}')
+    echo "- saving PCAP data spanning dates $START through $END"
+
+    # compare $START to $START_OLDEST
+    START_COMPARE=$(date -d $START +%s)
+    START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
+    if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
+      START_OLDEST=$START
+    fi  
+
+    # compare $ENDNEXT to $END_NEWEST
+    ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
+    ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
+    END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
+    if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
+      END_NEWEST=$ENDNEXT
+    fi  
+
+    cp -f "${PCAP}" "${PCAP_DIR}"/data.pcap
+
+  fi # end of valid pcap
+
+  echo
+
+done # end of for-loop processing pcap files
+
+# remove temp files
+echo "Cleaning up:"
+for TEMP_PCAP in ${TEMP_PCAPS[@]}; do
+  echo "- removing temporary pcap $TEMP_PCAP"
+  rm -f $TEMP_PCAP
+done
+
+# output final messages
+if [ "$INVALID_PCAPS" = "yes" ]; then
+  echo
+  echo "Please note!  One or more pcaps was invalid!  You can scroll up to see which ones were invalid."
+fi
+
+if [ "$VALID_PCAPS" = "yes" ]; then
+cat << EOF
+
+Import complete!
+
+You can use the following hyperlink to view data in the time range of your import.  You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
+https://{{ MASTERIP }}/kibana/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'${START_OLDEST}T00:00:00.000Z',mode:absolute,to:'${END_NEWEST}T00:00:00.000Z'))
+
+or you can manually set your Time Range to be:
+From: $START_OLDEST    To: $END_NEWEST
+
+Please note that it may take 30 seconds or more for events to appear in Kibana.
+EOF
+fi
--- a/salt/elasticsearch/files/ingest/zeek.dns
+++ b/salt/elasticsearch/files/ingest/zeek.dns
@@ -23,7 +23,7 @@
    { "rename": 	{ "field": "message2.TTLs", 		"target_field": "dns.ttls",			"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
    { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
-    { "pipeline": { "if": "ctx.dns.query.name.contains('.')", "name": "zeek.dns.tld"} },
+    { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "zeek.dns.tld" } },
    { "pipeline":       { "name": "zeek.common"											} }
  ]
 }
--- a/salt/elasticsearch/files/ingest/zeek.kerberos
+++ b/salt/elasticsearch/files/ingest/zeek.kerberos
@@ -13,9 +13,9 @@
    { "rename": 	{ "field": "message2.cipher", 		"target_field": "kerberos.ticket.cipher",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.forwardable", 	"target_field": "kerberos.ticket.forwardable",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.renewable", 	"target_field": "kerberos.ticket.renewable",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.client_cert_subject", 	"target_field": "kerberos.client.certificate.subject",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_cert_subject", 	"target_field": "kerberos.client_certificate_subject",	"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.client_cert_fuid", 	"target_field": "log.id.client_certificate_fuid",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.server_cert_subject", 	"target_field": "kerberos.server.certificate.subject",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_cert_subject", 	"target_field": "kerberos.server_certificate_subject",	"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.server_cert_fuid", 	"target_field": "log.id.server_certificate_fuid",	"ignore_missing": true 	} },
    { "pipeline":       { "name": "zeek.common"                                                                                   } }
  ]
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -118,12 +118,31 @@ filebeat.inputs:
    clean_removed: false
    close_removed: false

+  - type: log
+    paths:
+      - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log
+    fields:
+        module: zeek 
+        dataset: {{ LOGNAME }}
+        category: network
+        imported: true
+    processors:
+      - dissect:
+          tokenizer: "/nsm/import/%{import_id}/zeek/logs/%{import_source}"
+          field: "source"
+          target_prefix: ""
+      - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]
+ 
+    fields_under_root: true
+    clean_removed: false
+    close_removed: false
 {%- endfor %}
 {%- endif %}

  - type: log
    paths:
-      - /suricata/eve*.json
+      - /nsm/suricata/eve*.json
    fields:
      module: suricata
      dataset: common
@@ -137,8 +156,27 @@ filebeat.inputs:
    clean_removed: false
    close_removed: false

-  {%- if STRELKAENABLED == 1 %}
+  - type: log
+    paths:
+      - /nsm/import/*/suricata/eve*.json
+    fields:
+      module: suricata
+      dataset: common
+      category: network
+      imported: true 
+    processors:
+    - dissect:
+        tokenizer: "/nsm/import/%{import_id}/suricata/%{import_source}"
+        field: "source"
+        target_prefix: ""
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]

+    fields_under_root: true
+    clean_removed: false
+    close_removed: false
+
+  {%- if STRELKAENABLED == 1 %}
  - type: log
    paths:
      - /nsm/strelka/log/strelka.log
@@ -229,7 +267,7 @@ output.elasticsearch:
    - index: "so-strelka-%{+yyyy.MM.dd}"
      when.contains:
        module: "strelka"
-
+  
 setup.template.enabled: false
 {%- else %}

--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -55,9 +55,7 @@ so-filebeat:
    - binds:
      - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
      - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
-      - /nsm/zeek:/nsm/zeek:ro
-      - /nsm/strelka/log:/nsm/strelka/log:ro
-      - /nsm/suricata:/suricata:ro
+      - /nsm:/nsm:ro
      - /opt/so/wazuh/logs/alerts:/wazuh/alerts:ro
      - /opt/so/wazuh/logs/archives:/wazuh/archives:ro
      - /nsm/osquery/fleet/:/nsm/osquery/fleet:ro
--- a/salt/grafana/dashboards/standalone/standalone.json
+++ b/salt/grafana/dashboards/standalone/standalone.json
@@ -461,7 +461,7 @@
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
-      "title": "Panel Title{{ SERVERNAME }} - Kibana CPU",
+      "title": "{{ SERVERNAME }} - Kibana CPU",
      "tooltip": {
        "shared": true,
        "sort": 0,
--- a/salt/idstools/etc/disable.conf
+++ b/salt/idstools/etc/disable.conf
@@ -1,3 +1,4 @@
+{% set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%}
 # idstools - disable.conf

 # Example of disabling a rule by signature ID (gid is optional).
@@ -8,3 +9,7 @@
 # - All regular expression matches are case insensitive.
 # re:hearbleed
 # re:MS(0[7-9]|10)-\d+
+
+{%- for sid in disabled_sids %}
+{{ sid }}
+{%- endfor %}
--- a/salt/idstools/etc/enable.conf
+++ b/salt/idstools/etc/enable.conf
@@ -1,3 +1,4 @@
+{% set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%}
 # idstools-rulecat - enable.conf

 # Example of enabling a rule by signature ID (gid is optional).
@@ -8,3 +9,7 @@
 # - All regular expression matches are case insensitive.
 # re:hearbleed
 # re:MS(0[7-9]|10)-\d+
+
+{%- for sid in enabled_sids %}
+{{ sid }}
+{%- endfor %}
--- a/salt/idstools/init.sls
+++ b/salt/idstools/init.sls
@@ -66,3 +66,5 @@ so-idstools:
    - binds:
      - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
      - /opt/so/rules/nids:/opt/so/rules/nids:rw
+    - watch:
+      - file: idstoolsetcsync
--- a/salt/idstools/localrules/local.rules
+++ b/salt/idstools/localrules/local.rules
@@ -1 +1 @@
-# Put your own custom Snort/Suricata rules in here.
+# Put your own custom Snort/Suricata rules in /opt/so/saltstack/local/salt/idstools/localrules/.
--- a/salt/nginx/init.sls
+++ b/salt/nginx/init.sls
@@ -49,6 +49,16 @@ navigatorconfig:
    - makedirs: True
    - template: jinja

+navigatordefaultlayer:
+  file.managed:
+    - name: /opt/so/conf/navigator/nav_layer_playbook.json
+    - source: salt://nginx/files/nav_layer_playbook.json
+    - user: 939
+    - group: 939
+    - makedirs: True
+    - replace: False
+    - template: jinja
+
 so-nginx:
  docker_container.running:
    - image: {{ MASTER }}:5000/soshybridhunter/so-nginx:{{ VERSION }}
--- a/salt/soctopus/init.sls
+++ b/salt/soctopus/init.sls
@@ -49,16 +49,6 @@ playbookrulessync:
    - group: 939
    - template: jinja

-navigatordefaultlayer:
-  file.managed:
-    - name: /opt/so/conf/navigator/nav_layer_playbook.json
-    - source: salt://nginx/files/nav_layer_playbook.json
-    - user: 939
-    - group: 939
-    - makedirs: True
-    - replace: False
-    - template: jinja
-
 so-soctopus:
  docker_container.running:
    - image: {{ MASTER }}:5000/soshybridhunter/so-soctopus:{{ VERSION }}