Merge remote-tracking branch 'remotes/origin/dev' into issue/90

exclude-list.txt

@@ -1,2 +0,0 @@
-salt/bro/files/local.bro
-salt/bro/files/local.bro.community

salt/common/tools/sbin/soup

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+clone_to_tmp() {
+
+  # TODO Need to add a air gap option
+  # Make a temp location for the files
+  rm -rf /tmp/soup
+  mkdir -p /tmp/soup
+  cd /tmp/soup
+  #git clone -b dev https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
+  git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
+
+}
+
+# Prompt the user that this requires internets
+
+clone_to_tmp
+cd /tmp/soup/securityonion-saltstack/update
+chmod +x soup
+./soup
+
+

salt/elasticsearch/files/ingest/ossec.alert

@@ -8,14 +8,15 @@
     { "rename":         { "field": "message2.decoder",                          "target_field": "decoder",              "ignore_missing": true  } },
     { "rename":         { "field": "message2.full_log",                         "target_field": "log.full",             "ignore_missing": true  } },
     { "rename":         { "field": "message2.id",                               "target_field": "log.id.id",                    "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.location", 			"target_field": "location",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.location",                         "target_field": "log.location",         "ignore_missing": true  } },
     { "rename":         { "field": "message2.manager",                          "target_field": "manager",              "ignore_missing": true  } },
     { "rename":         { "field": "message2.predecoder",                       "target_field": "predecoder",           "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.timestamp", 			"target_field": "timestamp",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.timestamp",                        "target_field": "event.timestamp",              "ignore_missing": true  } },
     { "rename":         { "field": "message2.previous_log",                        "target_field": "log.previous_log",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.previous_output",                        "target_field": "log.previous_output",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.rule",                             "target_field": "rule",         "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.command", 				"target_field": "command",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.syscheck",                         "target_field": "host.syscheck",        "ignore_missing": true  } },
+    { "rename":         { "field": "data.command",                              "target_field": "process.command_line",         "ignore_missing": true  } },
     { "rename":         { "field": "data.dstip",                                "target_field": "destination.ip",       "ignore_missing": true  } },
     { "rename":         { "field": "data.dstport",                              "target_field": "destination.port",     "ignore_missing": true  } },
     { "rename":         { "field": "data.dstuser",                              "target_field": "user.escalated",       "ignore_missing": true  } },

salt/elasticsearch/files/ingest/zeek.weird

@@ -6,6 +6,9 @@
     { "rename": 	{ "field": "message2.addl", 		"target_field": "weird.additional_info",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.notice", 		"target_field": "weird.notice",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.peer", 		"target_field": "weird.peer",			"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.p",             "target_field": "weird.p",                   "ignore_missing": true  } },
+    { "rename":         { "field": "message2.dst",             "target_field": "destination.ip",                   "ignore_missing": true  } },
+    { "rename":         { "field": "message2.src",             "target_field": "source.ip",                   "ignore_missing": true  } },
     { "pipeline":       { "name": "zeek.common"                                                                                   } }
   ]
 }

salt/kibana/bin/so-kibana-config-load

@@ -0,0 +1,34 @@
+#!/bin/bash
+KIBANA_VERSION="7.6.1"
+MAX_WAIT=60
+
+# Check to see if Kibana is available
+wait_step=0
+  until curl -s -XGET http://localhost:5601 > /dev/null ; do
+  wait_step=$(( ${wait_step} + 1 ))
+  echo "Waiting on Kibana...Attempt #$wait_step"
+        if [ ${wait_step} -gt ${MAX_WAIT} ]; then
+                echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
+                exit 5
+        fi
+                  sleep 1s;
+  done
+
+# Sleep additional JIC server is not ready
+sleep 30s
+
+# Load config
+curl -X PUT "localhost:5601/api/saved_objects/config/$KIBANA_VERSION" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'
+{ "attributes":
+        {
+          "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29",
+          "defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645",
+          "discover:sampleSize":"100",
+          "dashboard:defaultDarkTheme":true,
+          "theme:darkMode":true,
+          "timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\",\n  \"mode\": \"quick\"\n}"
+        }
+}'
+
+# Load saved objects
+curl -X POST "localhost:5601/api/saved_objects/_import" -H "kbn-xsrf: true" --form file=@/opt/so/saltstack/salt/kibana/saved_objects.ndjson

salt/kibana/etc/config.json

@@ -1,6 +1,7 @@
 { "attributes":
         {
-    "defaultIndex": "*:so-*",
+          "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29",
+          "defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645",
           "discover:sampleSize":"100",
           "dashboard:defaultDarkTheme":true,
           "theme:darkMode":true,

salt/kibana/etc/kibana.yml

@@ -11,3 +11,4 @@ elasticsearch.hosts: [ "http://{{ ES }}:9200" ]
 #xpack.monitoring.ui.container.elasticsearch.enabled: true
 elasticsearch.requestTimeout: 90000
 logging.dest: /var/log/kibana/kibana.log
+telemetry.enabled: false

salt/kibana/init.sls

@@ -1,4 +1,4 @@
-{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %}
+{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.1') %}
 {% set MASTER = salt['grains.get']('master') %}
 {% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {% if FEATURES %}
@@ -59,6 +59,8 @@ synckibanacustom:
     - user: 932
     - group: 939
 
+# File.Recurse for custom saved dashboards
+
 # Start the kibana docker
 so-kibana:
   docker_container.running:
@@ -66,7 +68,6 @@ so-kibana:
     - hostname: kibana
     - user: kibana
     - environment:
-      - KIBANA_DEFAULTAPPID=dashboard/94b52620-342a-11e7-9d52-4f090484f59e
       - ELASTICSEARCH_HOST={{ MASTER }}
       - ELASTICSEARCH_PORT=9200
       - MASTER={{ MASTER }}
@@ -77,3 +78,17 @@ so-kibana:
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
     - port_bindings:
       - 0.0.0.0:5601:5601
+
+so-kibana-config-load:
+  cmd.script:
+    - shell: /bin/bash
+    - runas: socore
+    - source: salt://kibana/bin/so-kibana-config-load
+
+# Keep the setting correct
+#KibanaHappy:
+#  cmd.script:
+#    - shell: /bin/bash
+#    - runas: socore
+#    - source: salt://kibana/bin/keepkibanahappy.sh
+#    - template: jinja

salt/logstash/pipelines/templates/so/so-common-template.json

@@ -192,6 +192,14 @@
           "type":"object",
           "dynamic": true
         },
+	"message":{  
+          "type":"text",
+          "fields":{  
+            "keyword":{  
+              "type":"keyword"
+            }
+          }
+        },
         "modbus":{
           "type":"object",
           "dynamic": true

salt/registry/init.sls

@@ -27,15 +27,15 @@ dockerregistryconf:
     - source: salt://registry/etc/config.yml
 
 # Copy the registry script
-dockerregistrybuild:
+#dockerregistrybuild:
-  file.managed:
+#  file.managed:
-    - name: /opt/so/conf/docker-registry/so-buildregistry
+#    - name: /opt/so/conf/docker-registry/so-buildregistry
-    - source: salt://registry/bin/so-buildregistry
+#    - source: salt://registry/bin/so-buildregistry
-    - mode: 755
+#    - mode: 755
 
-dockerexpandregistry:
+#dockerexpandregistry:
- cmd.run:
+# cmd.run:
-   - name: /opt/so/conf/docker-registry/so-buildregistry
+#   - name: /opt/so/conf/docker-registry/so-buildregistry
 
 # Install the registry container
 so-dockerregistry:

setup/so-functions

@@ -17,6 +17,7 @@
 
 SCRIPTDIR=$(dirname "$0")
 source $SCRIPTDIR/so-whiptail
+SOVERSION=1.2.1
 
 accept_salt_key_local() {
   echo "Accept the key locally on the master" >> $SETUPLOG 2>&1
@@ -460,7 +461,7 @@ docker_registry() {
 
 }
 docker_seed_registry() {
-  VERSION="HH1.2.1"
+  VERSION="HH$SOVERSION"
   if [ $INSTALLTYPE != 'HELIXSENSOR' ]; then
     TRUSTED_CONTAINERS=( \
     "so-acng:$VERSION" \
@@ -743,7 +744,7 @@ master_static() {
   touch /opt/so/saltstack/pillar/static.sls
 
   echo "static:" > /opt/so/saltstack/pillar/static.sls
-  echo "  soversion: HH1.2.1" >> /opt/so/saltstack/pillar/static.sls
+  echo "  soversion: HH$SOVERSION" >> /opt/so/saltstack/pillar/static.sls
   echo "  hnmaster: $HNMASTER" >> /opt/so/saltstack/pillar/static.sls
   echo "  ntpserver: $NTPSERVER" >> /opt/so/saltstack/pillar/static.sls
   echo "  proxy: $PROXY" >> /opt/so/saltstack/pillar/static.sls
@@ -1280,8 +1281,8 @@ set_hostname() {
   echo "::1   localhost localhost.localdomain localhost6 localhost6.localdomain6" >> /etc/hosts
   echo $HOSTNAME > /etc/hostname
   HOSTNAME=$(cat /etc/hostname)
-  if [ $INSTALLTYPE != 'MASTER' ] || [ $INSTALLTYPE != 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then
+  if [ $INSTALLTYPE != 'MASTER' || $INSTALLTYPE != 'EVAL' || $INSTALLTYPE == 'HELIXSENSOR' || $INSTALLTYPE == 'MASTERSEARCH' ]; then
-    if [[ $TESTHOST = *"not found"* ]] || [[ $TESTHOST = *"connection timed out"* ]]; then
+    if [[ $TESTHOST = *"not found"* ]] || [ -z $TESTHOST ] || [[ $TESTHOST = *"connection timed out"* ]]; then
       if ! grep -q $MSRVIP /etc/hosts; then
         echo "$MSRVIP   $MSRV" >> /etc/hosts
       fi
@@ -1412,6 +1413,11 @@ set_updates() {
     fi
 }
 
+set_version() {
+  # Drop a file with the current version
+  echo "$SOVERSION" > /etc/soversion
+}
+
 update_sudoers() {
 
   if ! grep -qE '^socore\ ALL=\(ALL\)\ NOPASSWD:(\/usr\/bin\/salt\-key|\/opt\/so\/saltstack)' /etc/sudoers; then

setup/so-setup

@@ -159,6 +159,7 @@ if (whiptail_you_sure) ; then
     calculate_useable_cores
     whiptail_make_changes
     set_hostname
+    set_version
     clear_master
     mkdir -p /nsm
     get_filesystem_root
@@ -302,6 +303,7 @@ if (whiptail_you_sure) ; then
     # Last Chance to back out
     whiptail_make_changes
     set_hostname
+    set_version
     generate_passwords
     auth_pillar
     clear_master
@@ -570,6 +572,7 @@ if (whiptail_you_sure) ; then
     fi
     whiptail_make_changes
     set_hostname
+    set_version
     generate_passwords
     auth_pillar
     clear_master

so-setup-network

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
 
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by

updatemaster

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# Clone github
-mkdir /tmp/sogh
-cd /tmp/sogh
-#git clone -b dev https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
-git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
-cd securityonion-saltstack
-rsync -a --exclude-from 'exclude-list.txt' salt /opt/so/saltstack/
-chown -R socore:socore /opt/so/saltstack/salt
-chmod 755 /opt/so/saltstack/pillar/firewall/addfirewall.sh
-cd ~
-rm -rf /tmp/sogh
-# Run so-elastic-download here and call this soup with some magic
-salt-call state.highstate

upgrade/so-update-functions

@@ -0,0 +1,207 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Set the new SO Version
+
+UPDATEVERSION=1.2.1
+BUILD=HH
+
+#Determine the current install version
+
+if [ -f /etc/soversion ]; then
+  OLDVERSION=$(cat /etc/soversion)
+else
+  OLDVERSION=1.1.4
+fi
+
+# Use the hostname
+HOSTNAME=$(hostname)
+
+# List all the containers
+if [ $MASTERCHECK != 'so-helix' ]; then
+    TRUSTED_CONTAINERS=( \
+    "so-acng:$BUILD$UPDATEVERSION" \
+    "so-auth-api:$BUILD$UPDATEVERSION" \
+    "so-auth-ui:$BUILD$UPDATEVERSION" \
+    "so-core:$BUILD$UPDATEVERSION" \
+    "so-thehive-cortex:$BUILD$UPDATEVERSION" \
+    "so-curator:$BUILD$UPDATEVERSION" \
+    "so-domainstats:$BUILD$UPDATEVERSION" \
+    "so-elastalert:$BUILD$UPDATEVERSION" \
+    "so-elasticsearch:$BUILD$UPDATEVERSION" \
+    "so-filebeat:$BUILD$UPDATEVERSION" \
+    "so-fleet:$BUILD$UPDATEVERSION" \
+    "so-fleet-launcher:$BUILD$UPDATEVERSION" \
+    "so-freqserver:$BUILD$UPDATEVERSION" \
+    "so-grafana:$BUILD$UPDATEVERSION" \
+    "so-idstools:$BUILD$UPDATEVERSION" \
+    "so-influxdb:$BUILD$UPDATEVERSION" \
+    "so-kibana:$BUILD$UPDATEVERSION" \
+    "so-logstash:$BUILD$UPDATEVERSION" \
+    "so-mysql:$BUILD$UPDATEVERSION" \
+    "so-navigator:$BUILD$UPDATEVERSION" \
+    "so-playbook:$BUILD$UPDATEVERSION" \
+    "so-redis:$BUILD$UPDATEVERSION" \
+    "so-sensoroni:$BUILD$UPDATEVERSION" \
+    "so-soctopus:$BUILD$UPDATEVERSION" \
+    "so-steno:$BUILD$UPDATEVERSION" \
+    "so-strelka:$BUILD$UPDATEVERSION" \
+    "so-suricata:$BUILD$UPDATEVERSION" \
+    "so-telegraf:$BUILD$UPDATEVERSION" \
+    "so-thehive:$BUILD$UPDATEVERSION" \
+    "so-thehive-es:$BUILD$UPDATEVERSION" \
+    "so-wazuh:$BUILD$UPDATEVERSION" \
+    "so-zeek:$BUILD$UPDATEVERSION" )
+  else
+    TRUSTED_CONTAINERS=( \
+    "so-core:$BUILD$UPDATEVERSION" \
+    "so-filebeat:$BUILD$UPDATEVERSION" \
+    "so-idstools:$BUILD$UPDATEVERSION" \
+    "so-logstash:$BUILD$UPDATEVERSION" \
+    "so-redis:$BUILD$UPDATEVERSION" \
+    "so-sensoroni:$BUILD$UPDATEVERSION" \
+    "so-steno:$BUILD$UPDATEVERSION" \
+    "so-suricata:$BUILD$UPDATEVERSION" \
+    "so-telegraf:$BUILD$UPDATEVERSION" \
+    "so-zeek:$BUILD$UPDATEVERSION" )
+  fi
+
+
+clone_to_tmp() {
+
+  # TODO Need to add a air gap option
+  # Make a temp location for the files
+  mkdir /tmp/sogh
+  cd /tmp/sogh
+  #git clone -b dev https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
+  git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
+  cd /tmp
+
+}
+
+copy_new_files() {
+
+  # Copy new files over to the salt dir
+  cd /tmp/sogh/securityonion-saltstack
+  rsync -a --exclude-from 'exclude-list.txt' salt /opt/so/saltstack/
+  chown -R socore:socore /opt/so/saltstack/salt
+  chmod 755 /opt/so/saltstack/pillar/firewall/addfirewall.sh
+  cd /tmp
+}
+
+detect_os() {
+
+  # Detect Base OS
+  echo "Detecting Base OS" >> $UPDATELOG 2>&1
+  if [ -f /etc/redhat-release ]; then
+    OS=centos
+    if grep -q "CentOS Linux release 7" /etc/redhat-release; then
+      OSVER=7
+    elif grep -q "CentOS Linux release 8" /etc/redhat-release; then
+      OSVER=8
+      echo "We currently do not support CentOS $OSVER but we are working on it!"
+      exit
+    else
+      echo "We do not support the version of CentOS you are trying to use"
+      exit
+    fi
+
+  elif [ -f /etc/os-release ]; then
+    OS=ubuntu
+    if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
+      OSVER=bionic
+    elif grep -q "UBUNTU_CODENAME=xenial" /etc/os-release; then
+      OSVER=xenial
+    else
+      echo "We do not support your current version of Ubuntu"
+      exit
+    fi
+  else
+    echo "We were unable to determine if you are using a supported OS." >> $UPDATELOG 2>&1
+    exit
+  fi
+
+  echo "Found OS: $OS $OSVER" >> $UPDATELOG 2>&1
+
+}
+
+master_check() {
+  # Check to see if this is a master
+  MASTERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
+  if [ $MASTERCHECK == 'so-eval' OR $MASTERCHECK == 'so-master' OR $MASTERCHECK == 'so-mastersearch' ]; then
+    echo "This is a master. We can proceed"
+  else
+    echo "Please run soup on the master. The master controls all updates."
+    exit
+}
+
+salt_highstate() {
+
+  salt-call state.highstate
+
+}
+
+update_held_packages() {
+
+    if [ $OS == "centos" ]
+      SALTVER=2019.2.3
+      DOCKERVER=
+      yum -y --disableexcludes=all update salt-$SALTVER
+      yum -y --disableexcludes=all update docker-ce-$DOCKERVER
+    else
+      SALTVER=2019.2.3+ds-1
+      DOCKERVER=5:19.03.8~3-0~ubuntu-xenial
+    fi 
+
+}
+
+update_all_packages() {
+  
+  # Update all the things based on OS
+  if [ $OS == "centos" ]; then
+    yum -y update
+  else
+    apt -y update && apt -y upgrade
+  fi
+
+}
+
+update_docker_containers() {
+  
+  # Download the containers from the interwebs
+  for i in "${TRUSTED_CONTAINERS[@]}"
+  do
+    # Pull down the trusted docker image
+    echo "Downloading $i"
+    docker pull --disable-content-trust=false docker.io/soshybridhunter/$i
+    # Tag it with the new registry destination
+    docker tag soshybridhunter/$i $HOSTNAME:5000/soshybridhunter/$i
+    docker push $HOSTNAME:5000/soshybridhunter/$i
+  done
+
+  for i in "${TRUSTED_CONTAINERS[@]}"
+  do
+    echo "Removing $i locally"
+    docker rmi soshybridhunter/$i
+  done
+  
+}
+
+update_hh_version() {
+  # Change the version number in the static pillar
+
+}

upgrade/soup

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+SCRIPTDIR=$(dirname "$0")
+source $SCRIPTDIR/so-update-functions
+
+# Update Packages
+master_check
+update_all_packages
+update_held_packages
+
+
+