Merge remote-tracking branch 'origin/2.4/dev' into 2.4/ubuntu

salt/common/tools/sbin/so-elastic-clear

@@ -21,16 +21,34 @@ Security Onion Elastic Clear
   -y         Skip interactive mode
 EOF
 }
-while getopts "h:y" OPTION
+while getopts "h:cdely" OPTION
 do
         case $OPTION in
                 h)
                         usage
                         exit 0
                         ;;
-                
+                c)
-                y)
+			DELETE_CASES_DATA=1
+			SKIP=1
+                        ;;
+                d)
+			DONT_STOP_SERVICES=1
                         SKIP=1
+			;;
+                e)
+			DELETE_ELASTALERT_DATA=1
+                        SKIP=1
+                        ;;
+		l)
+			DELETE_LOG_DATA=1
+			SKIP=1
+			;;
+                y)
+                        DELETE_CASES_DATA=1
+                        DELETE_ELASTALERT_DATA=1
+			DELETE_LOG_DATA=1
+			SKIP=1
                         ;;
                 *)
                         usage
@@ -54,41 +72,83 @@ if [ $SKIP -ne 1 ]; then
         if [ "$INPUT" != "AGREE" ] ; then exit 0; fi
 fi
 
-# Check to see if Logstash are running
-LS_ENABLED=$(so-status | grep logstash)
-EA_ENABLED=$(so-status | grep elastalert)
 
-if [ ! -z "$LS_ENABLED" ]; then
+if [ -z "$DONT_STOP_SERVICES" ]; then
+	# Stop Elastic Agent
+	for i in $(pgrep elastic-agent | grep -v grep); do
+		kill -9 $i;
+	done 
 
-        /usr/sbin/so-logstash-stop
+	# Check to see if Elastic Fleet, Logstash, Elastalert are running
+	#EF_ENABLED=$(so-status | grep elastic-fleet)
+	LS_ENABLED=$(so-status | grep logstash)
+	EA_ENABLED=$(so-status | grep elastalert)
 
+	#if [ ! -z "$EF_ENABLED" ]; then
+        #	/usr/sbin/so-elastic-fleet-stop
+	#fi
+
+	if [ ! -z "$LS_ENABLED" ]; then
+        	/usr/sbin/so-logstash-stop
+	fi
+
+	if [ ! -z "$EA_ENABLED" ]; then
+        	/usr/sbin/so-elastalert-stop
+	fi
 fi
 
-if [ ! -z "$EA_ENABLED" ]; then
+if [ ! -z "$DELETE_CASES_DATA" ]; then
-
+        # Delete Cases data
-        /usr/sbin/so-elastalert-stop
+        echo "Deleting Cases data..."
-
+        INDXS=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index | grep "so-case")
+        for INDX in ${INDXS}
+        do
+                echo "Deleting $INDX"
+                /usr/sbin/so-elasticsearch-query ${INDX} -XDELETE > /dev/null 2>&1
+        done
 fi
 
-# Delete data
+# Delete Elastalert data
-echo "Deleting data..."
+if [ ! -z "$DELETE_ELASTALERT_DATA" ]; then
-
+        # Delete Elastalert data
-INDXS=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
+        echo "Deleting Elastalert data..."
-for INDX in ${INDXS}
+        INDXS=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index | grep "elastalert")
-do
+        for INDX in ${INDXS}
-    curl -K /opt/so/conf/elasticsearch/curl.config-XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
+        do
-done
+                echo "Deleting $INDX"
-
+                /usr/sbin/so-elasticsearch-query ${INDX} -XDELETE > /dev/null 2>&1
-#Start Logstash
+        done
-if [ ! -z "$LS_ENABLED" ]; then
-
-        /usr/sbin/so-logstash-start
-
 fi
 
-if [ ! -z "$EA_ENABLED" ]; then
+# Delete log data
-
+if [ ! -z "$DELETE_LOG_DATA" ]; then
-        /usr/sbin/so-elastalert-start
+	echo "Deleting log data ..."
-
+	DATASTREAMS=$(/usr/sbin/so-elasticsearch-query _data_stream | jq -r '.[] |.[].name')
+	for DATASTREAM in ${DATASTREAMS}
+	do
+		# Delete the data stream
+        	echo "Deleting $DATASTREAM..."
+        	/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} -XDELETE > /dev/null 2>&1
+	done
 fi
 
+if [ -z "$DONT_STOP_SERVICES" ]; then
+	#Start Logstash
+	if [ ! -z "$LS_ENABLED" ]; then
+        	/usr/sbin/so-logstash-start
+
+	fi
+
+	#Start Elastic Fleet
+	#if [ ! -z "$EF_ENABLED" ]; then
+        # 	/usr/sbin/so-elastic-fleet-start
+	#fi
+
+	#Start Elastalert
+	if [ ! -z "$EA_ENABLED" ]; then
+        	/usr/sbin/so-elastalert-start
+	fi
+
+	# Start Elastic Agent
+	/usr/bin/elastic-agent restart
+fi

salt/common/tools/sbin/so-elastic-fleet-restart

@@ -9,4 +9,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-restart elastic-fleet $1
+/usr/sbin/so-restart elasticfleet $1

salt/common/tools/sbin/so-elastic-fleet-start

@@ -9,4 +9,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-start elastic-fleet $1
+/usr/sbin/so-start elasticfleet $1

salt/common/tools/sbin/so-elastic-fleet-stop

@@ -9,4 +9,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop elastic-fleet $1
+/usr/sbin/so-stop elasticfleet $1

salt/elasticsearch/init.sls

@@ -177,6 +177,7 @@ esyml:
         ESCONFIG: {{ ESCONFIG }}
     - template: jinja
 
+{% if GLOBALS.role != "so-searchnode" %}
 escomponenttemplates:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/templates/component
@@ -219,6 +220,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
       - cmd: so-elasticsearch-templates
 {% endfor %}
 {% endif %}
+{% endif %}
 
 esroles:
   file.recurse:
@@ -363,6 +365,8 @@ append_so-elasticsearch_so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-elasticsearch
 
+{% if GLOBALS.role != "so-searchnode" %}
+
 so-es-cluster-settings:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-cluster-settings
@@ -406,7 +410,7 @@ so-elasticsearch-roles-load:
     - require:
       - docker_container: so-elasticsearch
       - file: es_sync_scripts
-
+{% endif %}
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/soc/files/bin/salt-relay.sh

@@ -6,7 +6,8 @@
 
 PIPE_OWNER=${PIPE_OWNER:-socore}
 PIPE_GROUP=${PIPE_GROUP:-socore}
-SOC_PIPE=${SOC_PIPE_REQUEST:-/opt/so/conf/soc/salt/pipe}
+SOC_PIPE=${SOC_PIPE:-/opt/so/conf/soc/salt/pipe}
+CMD_PREFIX=${CMD_PREFIX:-""}
 PATH=${PATH}:/usr/sbin
 
 function log() {
@@ -26,7 +27,7 @@ function make_pipe() {
 make_pipe "${SOC_PIPE}"
 
 function list_minions() {
-  response=$(so-minion -o=list)
+  response=$($CMD_PREFIX so-minion -o=list)
   exit_code=$?
   if [[ $exit_code -eq 0 ]]; then
     log "Successful command execution"
@@ -42,7 +43,7 @@ function manage_minion() {
   op=$(echo "$request" | jq -r .operation)
   id=$(echo "$request" | jq -r .id)
 
-  response=$(so-minion "-o=$op" "-m=$id")
+  response=$($CMD_PREFIX so-minion "-o=$op" "-m=$id")
   exit_code=$?
   if [[ exit_code -eq 0 ]]; then
     log "Successful command execution"
@@ -75,14 +76,14 @@ function manage_user() {
       add|enable|disable|delete)
         email=$(echo "$request" | jq -r .email)
         log "Performing user '$op' for user '$email'"
-        response=$(so-user "$op" --email "$email" --skip-sync)
+        response=$($CMD_PREFIX so-user "$op" --email "$email" --skip-sync)
         exit_code=$?
         ;;
       addrole|delrole)
         email=$(echo "$request" | jq -r .email)
         role=$(echo "$request" | jq -r .role)
         log "Performing '$op' for user '$email' with role '$role'"
-        response=$(so-user "$op" --email "$email" --role "$role" --skip-sync)
+        response=$($CMD_PREFIX so-user "$op" --email "$email" --role "$role" --skip-sync)
         exit_code=$?
         ;;
       password)
@@ -98,12 +99,12 @@ function manage_user() {
         lastName=$(echo "$request" | jq -r .lastName)
         note=$(echo "$request" | jq -r .note)
         log "Performing '$op' update for user '$email' with firstname '$firstName', lastname '$lastName', and note '$note'"
-        response=$(so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note")
+        response=$($CMD_PREFIX so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note")
         exit_code=$?
         ;;
       sync)
         log "Performing '$op'"
-        response=$(so-user "$op")
+        response=$($CMD_PREFIX so-user "$op")
         exit_code=$?
         ;;
       *)
@@ -142,17 +143,17 @@ function manage_salt() {
     state)
       log "Performing '$op' for '$state' on minion '$minion'"
       state=$(echo "$request" | jq -r .state)
-      response=$(salt --async "$minion" state.apply "$state" queue=True)
+      response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=True)
       exit_code=$?
       ;;
     highstate)
       log "Performing '$op' on minion $minion"
-      response=$(salt --async "$minion" state.highstate queue=True)
+      response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=True)
       exit_code=$?
       ;;
     activejobs)
+      response=$($CMD_PREFIX salt-run jobs.active -out json -l quiet)
       log "Querying active salt jobs"
-      response=$(salt-run jobs.active -out json -l quiet)
       $(echo "$response" > "${SOC_PIPE}")
       return
       ;;

salt/zeek/defaults.yaml

@@ -21,6 +21,7 @@ zeek:
       SpoolDir: /nsm/zeek/spool
       CfgDir: /opt/zeek/etc
       CompressLogs: 1
+      ZeekPort: 27760
     local:
       '@load':
         - misc/loaded-scripts

setup/so-functions

@@ -775,11 +775,12 @@ check_requirements() {
 }
 
 check_sos_appliance() {
-	title "Is this is an SOS Appliance?"
 	if [ -f "/etc/SOSMODEL" ]; then
-	  local MODEL=$(cat /etc/SOSMODEL)
+		local MODEL=$(cat /etc/SOSMODEL)
-	  info "Found SOS Model $MODEL"
+		info "Found SOS Model $MODEL"
-	  echo "sosmodel: $MODEL" >> /etc/salt/grains
+		echo "sosmodel: $MODEL" >> /etc/salt/grains
+	else
+		info "Not an appliance"
 	fi
 }
 

setup/so-setup

@@ -599,6 +599,7 @@ if ! [[ -f $install_opt_file ]]; then
 		# Start the master service
 		copy_salt_master_config
 		configure_minion "$minion_type"
+		check_sos_appliance
 
 		logCmd "salt-key -yd $MINION_ID"
 		logCmd "salt-call state.show_top"
@@ -661,6 +662,7 @@ if ! [[ -f $install_opt_file ]]; then
 		update_packages
 		saltify
 		configure_minion "$minion_type"
+		check_sos_appliance
 		drop_install_options
 		checkin_at_boot
 		logCmd "salt-call state.apply setup.highstate_cron --local --file-root=../salt/"