From 7b93f355e21d23435ee8c9899b7b6ecd3e62612a Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Wed, 25 Aug 2021 15:17:19 -0400
Subject: [PATCH 01/19] so-import-evtx - timestamp extraction

---
 salt/common/tools/sbin/so-import-evtx      | 175 +++++++++++++++++++++
 salt/elasticsearch/files/ingest/import.wel |   6 +
 2 files changed, 181 insertions(+)
 create mode 100644 salt/common/tools/sbin/so-import-evtx

diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx
new file mode 100644
index 000000000..fe53ee601
--- /dev/null
+++ b/salt/common/tools/sbin/so-import-evtx
@@ -0,0 +1,175 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- set VERSION = salt['pillar.get']('global:soversion') %}
+{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{%- set MANAGERIP = salt['pillar.get']('global:managerip') -%}
+{%- set URLBASE = salt['pillar.get']('global:url_base') %}
+{% set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
+{% set ES_PW = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+
+INDEX_DATE=$(date +'%Y.%m.%d')
+RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
+
+. /usr/sbin/so-common
+
+function usage {
+  cat << EOF
+Usage: $0 <evtx-file-1> [evtx-file-2] [evtx-file-*]
+
+Imports one or more evtx files into Security Onion. The evtx files will be analyzed and made available for review in the Security Onion toolset.
+EOF
+}
+
+
+function evtx2es() {
+  EVTX=$1
+  HASH=$2
+  
+  docker run --rm \
+    -v "$EVTX:/tmp/$RUNID.evtx" \
+    so-evtx2es02 \
+    --host {{ MANAGERIP }} --scheme https \
+    --index so-beats-$INDEX_DATE --pipeline import.wel \
+    --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 2>&1
+
+
+  docker run --rm \
+    -v "$EVTX:/tmp/import.evtx" \
+    -v "/nsm/import/evtx-end_newest:/tmp/newest" \
+    -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \
+    --entrypoint '/calculate_timestamp.sh' \
+    so-evtx2es02
+}
+
+# if no parameters supplied, display usage
+if [ $# -eq 0 ]; then
+  usage
+  exit 1
+fi
+
+# ensure this is a Manager node
+require_manager
+
+# verify that all parameters are files
+for i in "$@"; do
+  if ! [ -f "$i" ]; then
+    usage
+    echo "\"$i\" is not a valid file!"
+    exit 2
+  fi
+done
+
+# track if we have any valid or invalid pcaps
+INVALID_EVTXS="no"
+VALID_EVTXS="no"
+
+# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
+START_OLDEST="2050-12-31"
+END_NEWEST="1971-01-01"
+
+touch /nsm/import/evtx-start_oldest
+touch /nsm/import/evtx-end_newest
+
+echo $START_OLDEST > /nsm/import/evtx-start_oldest
+echo $END_NEWEST > /nsm/import/evtx-end_newest
+
+# paths must be quoted in case they include spaces
+for EVTX in "$@"; do
+  EVTX=$(/usr/bin/realpath "$EVTX")
+  echo "Processing Import: ${EVTX}"
+
+  # generate a unique hash to assist with dedupe checks
+  HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
+  HASH_DIR=/nsm/import/${HASH}
+  echo "- assigning unique identifier to import: $HASH"
+
+  if [ -d $HASH_DIR ]; then
+    echo "- this EVTX has already been imported; skipping"
+    INVALID_EVTXS="yes"
+  else
+    VALID_EVTXS="yes"
+
+    EVTX_DIR=$HASH_DIR/evtx
+    mkdir -p $EVTX_DIR
+
+    # generate IDS alerts and write them to standard pipeline
+    echo "- importing logs with evtx2es"
+    evtx2es "${EVTX}" $HASH
+
+    #START=$(pcapinfo "${EVTX}" -a |grep "First packet time:" | awk '{print $4}')
+    #END=$(pcapinfo "${EVTX}" -e |grep "Last packet time:" | awk '{print $4}')
+    #echo "- saving EVTX data spanning dates $START through $END"
+
+    # compare $START to $START_OLDEST
+    START=$(cat /nsm/import/evtx-start_oldest)
+    START_COMPARE=$(date -d $START +%s)
+    START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
+    if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
+      START_OLDEST=$START
+    fi  
+
+    # compare $ENDNEXT to $END_NEWEST
+    END=$(cat /nsm/import/evtx-end_newest)
+    ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
+    ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
+    END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
+    if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
+      END_NEWEST=$ENDNEXT
+    fi  
+
+    cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx
+    chmod 644 "${EVTX_DIR}"/data.evtx
+
+  fi # end of valid evtx
+
+  echo
+
+done # end of for-loop processing evtx files
+
+# remove temp files
+echo "Cleaning up:"
+for TEMP_EVTX in ${TEMP_EVTXS[@]}; do
+  echo "- removing temporary evtx $TEMP_EVTX"
+  rm -f $TEMP_EVTX
+done
+
+# output final messages
+if [ "$INVALID_EVTXS" = "yes" ]; then
+  echo
+  echo "Please note!  One or more evtx was invalid!  You can scroll up to see which ones were invalid."
+fi
+
+START_OLDEST_FORMATTED=`date +%Y-%m-%d --date="$START_OLDEST"`
+START_OLDEST_SLASH=$(echo $START_OLDEST_FORMATTED | sed -e 's/-/%2F/g')
+END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
+
+if [ "$VALID_EVTXS" = "yes" ]; then
+cat << EOF
+
+Import complete!
+
+You can use the following hyperlink to view data in the time range of your import.  You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
+https://{{ URLBASE }}/#/hunt?q=import.id:${RUNID}%20%7C%20groupby%20event.module%20event.dataset&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC
+
+or you can manually set your Time Range to be (in UTC):
+From: $START_OLDEST_FORMATTED    To: $END_NEWEST
+
+Please note that it may take 30 seconds or more for events to appear in Hunt.
+EOF
+fi
diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel
index 5a04324b7..771d47b5d 100644
--- a/salt/elasticsearch/files/ingest/import.wel
+++ b/salt/elasticsearch/files/ingest/import.wel
@@ -4,6 +4,12 @@
     { "remove":         { "field": ["event.created","timestamp", "winlog.event_data.UtcTime", "event_record_id"],     "ignore_failure": true                                               } },
     { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
     { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
+    {
+  "dissect": {
+    "field": "log.file.name",
+    "pattern" : "/tmp/%{import.id}.evtx"
+   }
+},
     { "pipeline":    { "name": "common" } }
   ]
 }
\ No newline at end of file

From 2bf471054bb3c6aa0afde1c430ac7933c21427ce Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 2 Sep 2021 11:46:18 -0400
Subject: [PATCH 02/19] Cloudtrail Event Fields

---
 salt/soc/files/soc/hunt.eventfields.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json
index 354aa31ce..fa5f41ded 100644
--- a/salt/soc/files/soc/hunt.eventfields.json
+++ b/salt/soc/files/soc/hunt.eventfields.json
@@ -44,5 +44,6 @@
           ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ],
           ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ],
           "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ],
-          "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ]
+          "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ],
+          ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "s        ource.ip", "source.geo.region_iso_code" ]
           }

From 1cee603ee4f56d4c3f1917f41122dbd79589a2f2 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 2 Sep 2021 13:24:04 -0400
Subject: [PATCH 03/19] Squid event fields

---
 salt/soc/files/soc/hunt.eventfields.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json
index fa5f41ded..9eac130e8 100644
--- a/salt/soc/files/soc/hunt.eventfields.json
+++ b/salt/soc/files/soc/hunt.eventfields.json
@@ -45,5 +45,6 @@
           ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ],
           "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ],
           "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ],
-          ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "s        ource.ip", "source.geo.region_iso_code" ]
+          ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ],
+          ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ]
           }

From 9f5dafd560c47b00b6bc71e2cc6bd3078d59975f Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 2 Sep 2021 13:48:18 -0400
Subject: [PATCH 04/19] More Event Fields

---
 salt/soc/files/soc/hunt.eventfields.json | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json
index 9eac130e8..e519e78b6 100644
--- a/salt/soc/files/soc/hunt.eventfields.json
+++ b/salt/soc/files/soc/hunt.eventfields.json
@@ -30,7 +30,7 @@
           "::software": ["soc_timestamp", "source.ip", "software.name", "software.type" ],
           "::ssh": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssh.version", "ssh.hassh_version", "ssh.direction", "ssh.client", "ssh.server", "log.id.uid" ],
           "::ssl": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssl.server_name", "ssl.certificate.subject", "ssl.validation_status", "ssl.version", "log.id.uid" ],
-          "::syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ],
+          ":zeek:syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ],
           "::tunnels": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "tunnel_type", "action", "log.id.uid" ],
           "::weird": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "weird.name", "log.id.uid" ],
           "::x509": ["soc_timestamp", "x509.certificate.subject", "x509.certificate.key.type", "x509.certificate.key.length", "x509.certificate.issuer", "log.id.fuid" ],
@@ -44,7 +44,9 @@
           ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ],
           ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ],
           "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ],
+          "::ossec": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ],
           "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ],
+          ":syslog:syslog": ["soc_timestamp", "host.name", "metadata.ip_address", "real_message", "syslog.priority", "syslog.application" ],
           ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ],
           ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ]
           }

From e3900606dc3c3fe3a83892f448cb96dab75d35ab Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Sat, 4 Sep 2021 10:42:18 -0400
Subject: [PATCH 05/19] Enable index sorting by default but allow it to be
 disabled

---
 salt/elasticsearch/templates/so/so-common-template.json | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json
index c9772b5db..4394ebb65 100644
--- a/salt/elasticsearch/templates/so/so-common-template.json
+++ b/salt/elasticsearch/templates/so/so-common-template.json
@@ -1,3 +1,4 @@
+{%- set INDEX_SORTING = salt['pillar.get']('elasticsearch:index_sorting', True) %}
 {
   "index_patterns": ["so-*"],
   "version":50001,
@@ -8,8 +9,10 @@
     "index.refresh_interval":"30s",
     "index.routing.allocation.require.box_type":"hot",
     "index.mapping.total_fields.limit": "1500",
+{%- if INDEX_SORTING is sameas true %}
     "index.sort.field": "@timestamp",
     "index.sort.order": "desc",
+{%- endif %}
     "analysis": {
       "analyzer": {
         "es_security_analyzer": {

From b8600be0f102c351c6260b44748ad69c79da6b17 Mon Sep 17 00:00:00 2001
From: weslambert <wlambertts@gmail.com>
Date: Wed, 8 Sep 2021 12:12:09 -0400
Subject: [PATCH 06/19] Incude server.publicBaseUrl

---
 salt/kibana/etc/kibana.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml
index 6fcafe68f..2819f301d 100644
--- a/salt/kibana/etc/kibana.yml
+++ b/salt/kibana/etc/kibana.yml
@@ -3,9 +3,11 @@
 {%- set ES = salt['pillar.get']('manager:mainip', '') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+{%- set URLBASE = salt['pillar.get']('global:url_base') %}
 server.name: kibana
 server.host: "0"
 server.basePath: /kibana
+server.publicBaseUrl: https://{{ URLBASE }}/kibana
 elasticsearch.hosts: [ "https://{{ ES }}:9200" ]
 elasticsearch.ssl.verificationMode: none
 #kibana.index: ".kibana"

From 5d235e932c02a2f3b06771c42c09e73cf5fc080b Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Thu, 9 Sep 2021 10:46:28 -0400
Subject: [PATCH 07/19] Fix Raid Status for cloud

---
 salt/common/tools/sbin/so-raid-status | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status
index e817554d3..54c2b3024 100755
--- a/salt/common/tools/sbin/so-raid-status
+++ b/salt/common/tools/sbin/so-raid-status
@@ -20,6 +20,10 @@
 appliance_check() {
   {%- if salt['grains.get']('sosmodel', '') %}
   APPLIANCE=1
+  {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %}
+  echo "nsmraid=0" > /opt/so/log/raid/status.log
+  exit 0
+  {%- endif %}
   DUDEYOUGOTADELL=$(dmidecode |grep Dell)
   if [[ -n $DUDEYOUGOTADELL ]]; then
   APPTYPE=dell

From c83f119cc08124dc41859ade5e01e00ba08e4bf6 Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Thu, 9 Sep 2021 10:59:35 -0400
Subject: [PATCH 08/19] Update so-raid-status

---
 salt/common/tools/sbin/so-raid-status | 1 -
 1 file changed, 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status
index 54c2b3024..4664a5961 100755
--- a/salt/common/tools/sbin/so-raid-status
+++ b/salt/common/tools/sbin/so-raid-status
@@ -21,7 +21,6 @@ appliance_check() {
   {%- if salt['grains.get']('sosmodel', '') %}
   APPLIANCE=1
   {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %}
-  echo "nsmraid=0" > /opt/so/log/raid/status.log
   exit 0
   {%- endif %}
   DUDEYOUGOTADELL=$(dmidecode |grep Dell)

From db50ef71b4297605b4d1eb7daef2b41c0d4c5cb6 Mon Sep 17 00:00:00 2001
From: doug <doug.burks@gmail.com>
Date: Fri, 10 Sep 2021 06:19:16 -0400
Subject: [PATCH 09/19] FEATURE: Add docs link to Setup #5459

---
 setup/so-whiptail | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/setup/so-whiptail b/setup/so-whiptail
index 1f0986119..ed4067da1 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -1867,8 +1867,9 @@ whiptail_you_sure() {
 	read -r -d '' you_sure_text <<- EOM
 		Welcome to Security Onion Setup!
 
-		You can use Setup for lots of different use cases from a small standalone installation to a large distributed deployment for your enterprise.
-		
+		You can use Setup for lots of different use cases from a small standalone installation to a large distributed deployment for your enterprise. Don't forget to review the documentation at:
+		https://docs.securityonion.net
+
 		Setup uses keyboard navigation and you can use arrow keys to move around.  Certain screens may provide a list and ask you to select one or more items from that list.  You can use [SPACE] to select items and [ENTER] to proceed to the next screen.
 		
 		Would you like to continue?

From 4f6a3269cb0e5c8c1378cf46c7b84bc389722b45 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Fri, 10 Sep 2021 09:59:47 -0400
Subject: [PATCH 10/19] Add more detail to syscollector

---
 salt/soc/files/soc/hunt.eventfields.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json
index e519e78b6..7964a360b 100644
--- a/salt/soc/files/soc/hunt.eventfields.json
+++ b/salt/soc/files/soc/hunt.eventfields.json
@@ -45,7 +45,7 @@
           ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ],
           "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ],
           "::ossec": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ],
-          "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ],
+          "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "log.full", "event.dataset", "event.module" ],
           ":syslog:syslog": ["soc_timestamp", "host.name", "metadata.ip_address", "real_message", "syslog.priority", "syslog.application" ],
           ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ],
           ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ]

From 9c310de459eeafab4913a8dd0533a53b0a5c5a17 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 10 Sep 2021 13:15:09 -0400
Subject: [PATCH 11/19] Add `--force` flag to idstools-rulecat under
 so-rule-update

This forces idstools to pull from the url each time, which prevents it from clearing all.rules if idstools-rulecat is run twice within 15 minutes by any method (either restarting the container or running so-rule-update)
---
 salt/common/tools/sbin/so-rule-update | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update
index 397719d61..a4e4d3cd7 100755
--- a/salt/common/tools/sbin/so-rule-update
+++ b/salt/common/tools/sbin/so-rule-update
@@ -10,4 +10,4 @@ got_root() {
 }
 
 got_root
-docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat $1"
+docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1"

From 0b42b19763093f3597a6905138571a0a5777d6c7 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 10 Sep 2021 13:41:58 -0400
Subject: [PATCH 12/19] Update so-rule-update to source so-common

---
 salt/common/tools/sbin/so-rule-update | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update
index a4e4d3cd7..0fc691d36 100755
--- a/salt/common/tools/sbin/so-rule-update
+++ b/salt/common/tools/sbin/so-rule-update
@@ -1,13 +1,7 @@
 #!/bin/bash
-got_root() {
 
-  # Make sure you are root
-  if [ "$(id -u)" -ne 0 ]; then
-          echo "This script must be run using sudo!"
-          exit 1
-  fi
+. /usr/sbin/so-common
 
-}
 
 got_root
 docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1"

From 679faddd52469395a2b0ece8ab64c24cd67584ba Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 10 Sep 2021 13:44:37 -0400
Subject: [PATCH 13/19] Update so-rule-update to pass all args to docker exec

Instead of passing $1, build a string from all args and add that to the command string for the docker exec statement
---
 salt/common/tools/sbin/so-rule-update | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update
index 0fc691d36..a3c2616a4 100755
--- a/salt/common/tools/sbin/so-rule-update
+++ b/salt/common/tools/sbin/so-rule-update
@@ -2,6 +2,9 @@
 
 . /usr/sbin/so-common
 
+argstr=""
+for arg in "$@"; do
+    argstr="${argstr} \"${arg}\""
+done
 
-got_root
-docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1"
+docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"

From 0b0d5085853ffba59e57f103eb23cdb34500b1e8 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 14 Sep 2021 12:01:14 -0400
Subject: [PATCH 14/19] so-import-evtx - tweaks

---
 salt/common/tools/sbin/so-import-evtx | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx
index fe53ee601..e860881f3 100644
--- a/salt/common/tools/sbin/so-import-evtx
+++ b/salt/common/tools/sbin/so-import-evtx
@@ -46,7 +46,7 @@ function evtx2es() {
     so-evtx2es02 \
     --host {{ MANAGERIP }} --scheme https \
     --index so-beats-$INDEX_DATE --pipeline import.wel \
-    --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 2>&1
+    --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null
 
 
   docker run --rm \
@@ -75,7 +75,7 @@ for i in "$@"; do
   fi
 done
 
-# track if we have any valid or invalid pcaps
+# track if we have any valid or invalid evtx
 INVALID_EVTXS="no"
 VALID_EVTXS="no"
 
@@ -108,14 +108,10 @@ for EVTX in "$@"; do
     EVTX_DIR=$HASH_DIR/evtx
     mkdir -p $EVTX_DIR
 
-    # generate IDS alerts and write them to standard pipeline
+    # import evtx and write them to import ingest pipeline
     echo "- importing logs with evtx2es"
     evtx2es "${EVTX}" $HASH
 
-    #START=$(pcapinfo "${EVTX}" -a |grep "First packet time:" | awk '{print $4}')
-    #END=$(pcapinfo "${EVTX}" -e |grep "Last packet time:" | awk '{print $4}')
-    #echo "- saving EVTX data spanning dates $START through $END"
-
     # compare $START to $START_OLDEST
     START=$(cat /nsm/import/evtx-start_oldest)
     START_COMPARE=$(date -d $START +%s)

From 74b0b365bdd7ce28206df6675686008ea3d86537 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 14 Sep 2021 13:23:07 -0400
Subject: [PATCH 15/19] Fleet SA - SOC Link Fix

---
 salt/soc/files/soc/soc.json | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index 6119f0e6b..4798ef31e 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -3,7 +3,8 @@
 {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
 {%- set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
 {%- set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{%- set OSQUERY = salt['pillar.get']('manager:osquery', '0') %}
+{% set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %}
+{% set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %}
 {%- set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
 {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
 {%- set API_TIMEOUT = salt['pillar.get']('sensoroni:api_timeout_ms', 0) %}
@@ -116,7 +117,7 @@
         {%- if THEHIVE == 0 %}
           "toolTheHive",
         {%- endif %}
-        {%- if OSQUERY == 0 %}
+        {%- if FLEETMANAGER or FLEETNODE %}
           "toolFleet",
         {%- endif %}
         {%- if GRAFANA == 0 %}

From e0a289182f643556fd031f6cfd8c26c07f90c3ed Mon Sep 17 00:00:00 2001
From: Josh Brower <josh@defensivedepth.com>
Date: Wed, 15 Sep 2021 09:28:23 -0400
Subject: [PATCH 16/19] Fix Fleet Link Logic

---
 salt/soc/files/soc/soc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index 4798ef31e..e9dfa50d8 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -117,7 +117,7 @@
         {%- if THEHIVE == 0 %}
           "toolTheHive",
         {%- endif %}
-        {%- if FLEETMANAGER or FLEETNODE %}
+        {%- if not FLEETMANAGER and not FLEETNODE %}
           "toolFleet",
         {%- endif %}
         {%- if GRAFANA == 0 %}

From aff571faf276d1856ee4cb80b6c56242410fd2a7 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 15 Sep 2021 13:32:52 -0400
Subject: [PATCH 17/19] soup changes 2.3.80

---
 salt/common/tools/sbin/soup | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 17339fdd5..c8935969d 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -385,6 +385,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
     [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
+    [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70]] && up_2.3.5X_to_2.3.80
     true
 }
 
@@ -622,6 +623,20 @@ EOF
   INSTALLEDVERSION=2.3.50
 }
 
+up_2.3.5X_to_2.3.80() {
+  
+  # Remove watermark settings from global.sls
+  sed -i '/  cluster_routing_allocation_disk/d' /opt/so/saltstack/local/pillar/global.sls
+
+  # Add new indices to the global
+  sed -i '/  index_settings:/a \\    so-elasticsearch: \n      shards: 1 \n      warm: 7 \n      close: 30 \n      delete: 365' /opt/so/saltstack/local/pillar/global.sls
+  sed -i '/  index_settings:/a \\    so-logstash: \n      shards: 1 \n      warm: 7 \n      close: 30 \n      delete: 365' /opt/so/saltstack/local/pillar/global.sls
+  sed -i '/  index_settings:/a \\    so-kibana: \n      shards: 1 \n      warm: 7 \n      close: 30 \n      delete: 365' /opt/so/saltstack/local/pillar/global.sls
+  sed -i '/  index_settings:/a \\    so-redis: \n      shards: 1 \n      warm: 7 \n      close: 30 \n      delete: 365' /opt/so/saltstack/local/pillar/global.sls
+  
+  INSTALLEDVERSION=2.3.80
+}
+
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then

From ac417867ed0c41c9eec7afa375ab793b201cc200 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Wed, 15 Sep 2021 14:06:08 -0400
Subject: [PATCH 18/19] so-import-evtx - final fixes

---
 salt/common/tools/sbin/so-import-evtx | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx
index e860881f3..9e640beaa 100644
--- a/salt/common/tools/sbin/so-import-evtx
+++ b/salt/common/tools/sbin/so-import-evtx
@@ -43,18 +43,19 @@ function evtx2es() {
   
   docker run --rm \
     -v "$EVTX:/tmp/$RUNID.evtx" \
-    so-evtx2es02 \
+    --entrypoint evtx2es \
+    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} \
     --host {{ MANAGERIP }} --scheme https \
     --index so-beats-$INDEX_DATE --pipeline import.wel \
-    --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null
-
+    --login {{ES_USER}} --pwd {{ES_PW}} \
+    "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null
 
   docker run --rm \
     -v "$EVTX:/tmp/import.evtx" \
     -v "/nsm/import/evtx-end_newest:/tmp/newest" \
     -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \
-    --entrypoint '/calculate_timestamp.sh' \
-    so-evtx2es02
+    --entrypoint '/evtx_calc_timestamps.sh' \
+    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}
 }
 
 # if no parameters supplied, display usage
@@ -109,7 +110,7 @@ for EVTX in "$@"; do
     mkdir -p $EVTX_DIR
 
     # import evtx and write them to import ingest pipeline
-    echo "- importing logs with evtx2es"
+    echo "- importing logs to Elasticsearch..."
     evtx2es "${EVTX}" $HASH
 
     # compare $START to $START_OLDEST

From a75238bc3fe8a654b010606657e342a05cd5227e Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Wed, 15 Sep 2021 14:13:16 -0400
Subject: [PATCH 19/19] so-import-evtx - fix ingest formatting

---
 salt/elasticsearch/files/ingest/import.wel | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel
index 771d47b5d..e75098f8f 100644
--- a/salt/elasticsearch/files/ingest/import.wel
+++ b/salt/elasticsearch/files/ingest/import.wel
@@ -4,12 +4,7 @@
     { "remove":         { "field": ["event.created","timestamp", "winlog.event_data.UtcTime", "event_record_id"],     "ignore_failure": true                                               } },
     { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
     { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
-    {
-  "dissect": {
-    "field": "log.file.name",
-    "pattern" : "/tmp/%{import.id}.evtx"
-   }
-},
+    { "dissect":       { "field": "log.file.name", "pattern" : "/tmp/%{import.id}.evtx" } },
     { "pipeline":    { "name": "common" } }
   ]
 }
\ No newline at end of file