Merge branch 'dev' into foxtrot

# Conflicts:
#	salt/common/tools/sbin/soup

pillar/logstash/manager.sls

@@ -3,7 +3,6 @@ logstash:
   pipelines:
     manager:
       config:
-        - so/0008_input_fleet_livequery.conf.jinja
         - so/0009_input_beats.conf      
         - so/0010_input_hhbeats.conf
         - so/9999_output_redis.conf.jinja

pillar/logstash/search.sls

@@ -8,7 +8,6 @@ logstash:
         - so/9002_output_import.conf.jinja
         - so/9034_output_syslog.conf.jinja
         - so/9100_output_osquery.conf.jinja  
-        - so/9101_output_osquery_livequery.conf.jinja        
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja

salt/common/tools/sbin/so-common

@@ -86,6 +86,19 @@ add_interface_bond0() {
 	fi
 }
 
+check_airgap() {
+  # See if this is an airgap install
+  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
+  if [[ "$AIRGAP" == "True" ]]; then
+      is_airgap=0
+      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
+      AGDOCKER=/tmp/soagupdate/docker
+      AGREPO=/tmp/soagupdate/Packages
+  else 
+      is_airgap=1
+  fi
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -97,6 +110,46 @@ check_password() {
 	return $?
 }
 
+check_elastic_license() {
+
+	[ -n "$TESTING" ] && return
+
+	# See if the user has already accepted the license
+	if [ ! -f /opt/so/state/yeselastic.txt ]; then
+	  elastic_license
+	else
+	  echo "Elastic License has already been accepted"
+	fi  
+}
+
+elastic_license() {
+
+read -r -d '' message <<- EOM
+\n
+Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
+https://securityonion.net/elastic-license
+
+Please review the Elastic License:
+https://www.elastic.co/licensing/elastic-license
+
+Do you agree to the terms of the Elastic License?
+
+If so, type AGREE to accept the Elastic License and continue.  Otherwise, press Enter to exit this program without making any changes.
+EOM
+
+AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
+"$message" 20 75 3>&1 1>&2 2>&3)
+
+if [ "$AGREED" = 'AGREE' ]; then
+  mkdir -p /opt/so/state
+  touch /opt/so/state/yeselastic.txt 
+else
+  echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+  exit 1
+fi
+
+}
+
 fail() {
 	msg=$1
 	echo "ERROR: $msg"
@@ -250,6 +303,12 @@ set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 
+set_palette() {
+	if [ "$OS" == ubuntu ]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}
+
 set_version() {
 	CURRENTVERSION=0.0.0
 	if [ -f /etc/soversion ]; then

salt/common/tools/sbin/soup

@@ -19,13 +19,12 @@
 
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
+POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
 
-exec 3>&1 1>${SOUP_LOG} 2>&1
-
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
   cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -101,19 +100,6 @@ update_registry() {
   salt-call state.apply registry queue=True
 }
 
-check_airgap() {
-  # See if this is an airgap install
-  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
-  if [[ "$AIRGAP" == "True" ]]; then
-    is_airgap=0
-    UPDATE_DIR=/tmp/soagupdate/SecurityOnion
-    AGDOCKER=/tmp/soagupdate/docker
-    AGREPO=/tmp/soagupdate/Packages
-  else 
-    is_airgap=1
-  fi
-}
-
 check_sudoers() {
 	if grep -q "so-setup" /etc/sudoers; then
 		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -243,21 +229,9 @@ masterunlock() {
   fi
 }
 
-playbook() {
+preupgrade_changes() {
-  echo "Applying playbook settings"
-  if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then
-    salt-call state.apply playbook.OLD_db_init
-    rm -f /opt/so/rules/elastalert/playbook/*.yaml
-    so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
-  fi
-  if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then
-    so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
-  fi
-}
-
-pillar_changes() {
     # This function is to add any new pillar items if needed.
-  echo "Checking to see if pillar changes are needed."
+    echo "Checking to see if changes are needed."
 
     [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
     [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
@@ -266,6 +240,26 @@ pillar_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
 }
 
+postupgrade_changes() {
+    # This function is to add any new pillar items if needed.
+    echo "Running post upgrade processes."
+    
+    [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
+    [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
+}
+
+post_rc1_to_2.3.21() {
+  salt-call state.apply playbook.OLD_db_init
+  rm -f /opt/so/rules/elastalert/playbook/*.yaml
+  so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
+  POSTVERSION=2.3.21
+}
+
+post_2.3.2X_to_2.3.30() {
+  so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
+  POSTVERSION=2.3.30
+}
+
 rc1_to_rc2() {
 
   # Move the static file to global.sls
@@ -408,16 +402,26 @@ up_2.3.2X_to_2.3.30() {
   check_log_size_limit
 }
 
-space_check() {
+verify_upgradespace() {
-  # Check to see if there is enough space
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then
-    echo "You are low on disk space. Upgrade will try and clean up space.";
+      echo "You are low on disk space."
-    clean_dockers
+      return 1
   else
-    echo "Plenty of space for upgrading"
+      return 0
   fi
+}
 
+upgrade_space() {
+  if ! verify_upgradespace; then
+    clean_dockers
+    if ! verify_upgradespace; then
+      echo "There is not enough space to perform the upgrade. Please free up space and try again"
+      exit 1
+    fi
+  else
+      echo "You have enough space for upgrade. Proceeding with soup."
+  fi  
 }
 
 thehive_maint() {
@@ -546,6 +550,7 @@ verify_latest_update_script() {
 }
 
 main () {
+echo "### Preparing soup at `date` ###"
 while getopts ":b" opt; do
   case "$opt" in
     b ) # process option b
@@ -572,6 +577,8 @@ check_airgap
 echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
 echo ""
 set_os
+set_palette
+check_elastic_license
 echo ""
 if [ $is_airgap -eq 0 ]; then
   # Let's mount the ISO since this is airgap
@@ -598,7 +605,7 @@ fi
 
 echo "Let's see if we need to update Security Onion."
 upgrade_check
-space_check
+upgrade_space
 
 echo "Checking for Salt Master and Minion updates."
 upgrade_check_salt
@@ -648,8 +655,7 @@ else
   echo ""
 fi
 
-echo "Making pillar changes."
+preupgrade_changes
-pillar_changes
 echo ""
 
 if [ $is_airgap -eq 0 ]; then
@@ -703,7 +709,7 @@ echo "Starting Salt Master service."
 systemctl start salt-master
 echo "Running a highstate. This could take several minutes."
 salt-call state.highstate -l info queue=True
-playbook
+postupgrade_changes
 unmount_update
 thehive_maint
 
@@ -740,6 +746,7 @@ NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
 if [ $NUM_MINIONS -gt 1 ]; then
 
 	cat << EOF
+	
 This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
@@ -750,7 +757,23 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
 EOF
 
 fi
-
+echo "### soup has been served at `date` ###"
 }
 
-main "$@" | tee /dev/fd/3
+cat << EOF
+
+SOUP - Security Onion UPdater
+
+Please review the following for more information about the update process and recent updates:
+https://docs.securityonion.net/soup
+https://blog.securityonion.net
+
+Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup.
+
+Press Enter to continue or Ctrl-C to cancel.
+
+EOF
+
+read input
+
+main "$@" | tee -a $SOUP_LOG

salt/elasticsearch/files/ingest/common

@@ -32,8 +32,6 @@
     { "rename":          { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
     { "rename":          { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
     { "lowercase":       { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },
-    { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },

salt/elasticsearch/templates/so/so-common-template.json

@@ -51,16 +51,29 @@
             "match_mapping_type": "string",
             "path_match": "*.ip",
             "mapping": {
-              "type": "ip"
+              "type": "ip",
+              "fields" : {
+                "keyword" : {
+                  "ignore_above" : 45,
+                  "type" : "keyword"
+                }
+              }
+
             }
           }
        },
        {
           "port": {
-            "match_mapping_type": "string",
             "path_match": "*.port",
             "mapping": {
-              "type": "integer"
+              "type": "integer",
+              "fields" : {
+                "keyword" : {
+                  "ignore_above" : 6,
+                  "type" : "keyword"
+                }
+              }
+
             }
           }
        },

salt/firewall/init.sls

@@ -26,15 +26,6 @@ iptables_fix_fwd:
     - position: 1
     - target: DOCKER-USER
 
-# Allow related/established sessions
-iptables_allow_established:
-  iptables.append:
-    - table: filter
-    - chain: INPUT
-    - jump: ACCEPT
-    - match: conntrack
-    - ctstate: 'RELATED,ESTABLISHED'
-
 # I like pings
 iptables_allow_pings:
   iptables.append:
@@ -77,17 +68,6 @@ enable_docker_user_fw_policy:
     - out-interface: docker0
     - position: 1
 
-enable_docker_user_established:
-  iptables.insert:
-    - table: filter
-    - chain: DOCKER-USER
-    - jump: ACCEPT
-    - in-interface: '!docker0'
-    - out-interface: docker0
-    - position: 1
-    - match: conntrack
-    - ctstate: 'RELATED,ESTABLISHED'
-
 {% set count = namespace(value=0) %}
 {% for chain, hg in assigned_hostgroups.chain.items() %}
   {% for hostgroup, portgroups in assigned_hostgroups.chain[chain].hostgroups.items() %}
@@ -120,6 +100,27 @@ enable_docker_user_established:
   {% endfor %}
 {% endfor %}
 
+# Allow related/established sessions
+iptables_allow_established:
+  iptables.insert:
+    - table: filter
+    - chain: INPUT
+    - jump: ACCEPT
+    - position: 1
+    - match: conntrack
+    - ctstate: 'RELATED,ESTABLISHED'
+
+enable_docker_user_established:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - in-interface: '!docker0'
+    - out-interface: docker0
+    - position: 1
+    - match: conntrack
+    - ctstate: 'RELATED,ESTABLISHED'
+
 # Block icmp timestamp response
 block_icmp_timestamp_reply:
   iptables.append:

salt/kibana/bin/so-kibana-config-load

@@ -24,3 +24,9 @@ THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}
 
 # Load saved objects
 curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
+
+# Disable certain Features from showing up in the Kibana UI
+
+echo
+echo "Setting up default Space:"
+curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' | jq

salt/strelka/files/filestream/filestream.yaml

@@ -19,7 +19,8 @@ files:
     - '/nsm/strelka/unprocessed/*'
   delete: false
   gatekeeper: true
+  processed: '/nsm/strelka/processed'
 response:
   report: 5s
 delta: 5s
-staging: '/nsm/strelka/processed'
+staging: '/nsm/strelka/staging'

salt/strelka/init.sls

@@ -86,6 +86,13 @@ strelkaprocessed:
     - group: 939
     - makedirs: True
 
+strelkastaging:
+   file.directory:
+    - name: /nsm/strelka/staging
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 strelkaunprocessed:
    file.directory:
     - name: /nsm/strelka/unprocessed
@@ -96,7 +103,7 @@ strelkaunprocessed:
 # Check to see if Strelka frontend port is available
 strelkaportavailable:
     cmd.run:
-      - name: netstat -utanp | grep ":57314" | grep -qv docker && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+      - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
 
 strelka_coordinator:
   docker_container.running:

salt/telegraf/etc/telegraf.conf

@@ -618,11 +618,8 @@
 # # Read stats from one or more Elasticsearch servers or clusters
 {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone']   %}
  [[inputs.elasticsearch]]
-
-#   ## specify a list of one or more Elasticsearch servers
-#   # you can add username and password to your url to use basic authentication:
-#   # servers = ["http://user:pass@localhost:9200"]
    servers = ["https://{{ MANAGER }}:9200"]
+   insecure_skip_verify = true
 {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode']  %}
  [[inputs.elasticsearch]]
    servers = ["https://{{ NODEIP }}:9200"]

setup/so-functions

@@ -1243,8 +1243,13 @@ es_heapsize() {
 		# https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
 		ES_HEAP_SIZE="25000m"
 	else
-		# Set heap size to 25% of available memory
+		# Set heap size to 33% of available memory
-		ES_HEAP_SIZE=$(( total_mem / 4 ))"m"
+		ES_HEAP_SIZE=$(( total_mem / 3 ))
+		if [ "$ES_HEAP_SIZE" -ge 25001 ] ; then
+		    ES_HEAP_SIZE="25000m"
+		else
+		    ES_HEAP_SIZE=$ES_HEAP_SIZE"m"
+		fi
 	fi
 	export ES_HEAP_SIZE
 
@@ -1686,8 +1691,8 @@ manager_global() {
 		"    so-zeek:"\
 		"      shards: 5"\
 		"      warm: 7"\
-		"      close: 365"\
+		"      close: 45"\
-		"      delete: 45"\
+		"      delete: 365"\
 		"minio:"\
 		"  access_key: '$ACCESS_KEY'"\
 		"  access_secret: '$ACCESS_SECRET'"\

setup/so-setup

@@ -168,10 +168,8 @@ local_sbin="$(pwd)/../salt/common/tools/sbin"
 export PATH=$PATH:$local_sbin
 
 set_network_dev_status_list
+set_palette >> $setup_log 2>&1
 
-if [ "$OS" == ubuntu ]; then
-	update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1
-fi
 
 # Kernel messages can overwrite whiptail screen #812
 # https://github.com/Security-Onion-Solutions/securityonion/issues/812
@@ -274,6 +272,10 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then
 	fi
 fi
 
+if [[ $is_manager || $is_import ]]; then
+	check_elastic_license
+fi
+
 if ! [[ -f $install_opt_file ]]; then
 	if [[ $is_manager && $is_sensor ]]; then
 		check_requirements "standalone"