Merge remote-tracking branch 'origin/2.4/dev' into 2.4/zeekbpf

salt/allowed_states.map.jinja

@@ -34,7 +34,7 @@
           'influxdb',
           'soc',
           'kratos',
-          'elastic-fleet',
+          'elasticfleet',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -105,7 +105,7 @@
           'schedule',
           'tcpreplay',
           'docker_clean',
-          'elastic-fleet'
+          'elasticfleet'
           ],
       'so-manager': [
           'salt.master',
@@ -118,7 +118,7 @@
           'influxdb',
           'soc',
           'kratos',
-          'elastic-fleet',
+          'elasticfleet',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -137,7 +137,7 @@
           'influxdb',
           'soc',
           'kratos',
-          'elastic-fleet',
+          'elasticfleet',
           'firewall',
           'manager',
           'idstools',
@@ -166,7 +166,7 @@
           'influxdb',
           'soc',
           'kratos',
-          'elastic-fleet',
+          'elasticfleet',
           'firewall',
           'idstools',
           'suricata.manager',

salt/common/tools/sbin/so-elastic-agent-gen-installers

@@ -24,11 +24,11 @@ mkdir -p /tmp/elastic-agent-workspace
 for OS in "${CONTAINERGOOS[@]}"
 do
     printf "\n\nGenerating $OS Installer..."
-    cp /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz
+    cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz
     docker run -e CGO_ENABLED=0 -e GOOS=$OS \
     --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/tmp/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elastic-fleet/files/so_agent-installers/,target=/output/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_$OS
     printf "\n $OS Installer Generated..."
 done

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -17,7 +17,9 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 
 # Disable certain Features from showing up in the Kibana UI
 echo
-echo "Setting up default Security Onion package policies for Elastic Agent..."
+echo "Disable certain Features from showing up in the Kibana UI"
+so-kibana-space-defaults
+echo
 
 # Suricata logs
 echo
@@ -71,7 +73,7 @@ echo
 # Kratos logs
 echo
 echo "Setting up Kratos package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- rename:\n    fields:\n      - from: \"audience\"\n        to: \"event.dataset\"\n    ignore_missing: true\n- add_fields:\n    when:\n      not: \n        has_fields: ['event.dataset']\n    target: ''\n    fields:\n      event.dataset: access", "custom": "pipeline: kratos" }}}}}}'
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos" }}}}}}'
 echo
 
 # RITA Logs

salt/common/tools/sbin/so-elastic-fleet-setup

@@ -91,19 +91,19 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
-salt-call state.apply elastic-fleet queue=True
+salt-call state.apply elasticfleet queue=True
 
 # Load Elastic Fleet integrations
 /usr/sbin/so-elastic-fleet-integration-policy-load
 
 # Temp
-wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/  https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz
+wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/  https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz
-wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz
+wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz
-wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-windows-x86_64.tar.gz
+wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-windows-x86_64.tar.gz
 
 #git clone -b 2.4-so-elastic-agent https://github.com/Security-Onion-Solutions/securityonion-image.git
 #cd securityonion-image/so-elastic-agent-builder
 #docker build -t so-elastic-agent-builder .
 
 so-elastic-agent-gen-installers
-salt-call state.apply elastic-fleet.install_agent_grid queue=True
+salt-call state.apply elasticfleet.install_agent_grid queue=True

salt/common/tools/sbin/so-kibana-space-defaults

@@ -13,6 +13,6 @@ echo "Setting up default Space:"
 {% if HIGHLANDER %}
 curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log
 {% else %}
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet","fleetv2","securitySolutionCases"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCases"]} ' >> /opt/so/log/kibana/misc.log
 {% endif %}
 echo

salt/common/tools/sbin/so-minion

@@ -23,7 +23,7 @@ if [[ $# -lt 1 ]]; then
   echo "   accept: Accepts a new key and adds the minion files"
   echo "   delete: Removes the key and deletes the minion files"
   echo "   reject: Rejects a key"
-  echo "     test: Ingest test data"
+  echo "     test: Perform minion test"
   echo ""
   exit 1
 fi

salt/elasticfleet/install_agent_grid.sls

@@ -9,7 +9,7 @@
 
 run_installer:
   cmd.script:
-    - name: salt://elastic-fleet/files/so_agent-installers/so-elastic-agent_linux
+    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux
     - args: -token={{ GRIDNODETOKEN }}
 
 {% endif %}

salt/elasticfleet/soc_elasticfleet.yaml

@@ -0,0 +1,18 @@
+elasticfleet:
+  server:
+    endpoints_enrollment:
+      description: Endpoint enrollment key.
+      global: True
+      helpLink: elastic-fleet.html
+    es_token:
+      description: Elastic auth token.
+      global: True
+      helpLink: elastic-fleet.html
+    grid_enrollment:
+      description: Grid enrollment key.
+      global: True
+      helpLink: elastic-fleet.html
+    url:
+      description: Agent connection URL.
+      global: True
+      helpLink: elastic-fleet.html

salt/elasticsearch/files/ingest/kratos

@@ -1,7 +1,9 @@
 {
   "description" : "kratos",
   "processors" : [
-    { "set": { "field": "event.dataset", "value": "access" } },
+    {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
+    {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"kratos.{{{audience}}}","media_type":"text/plain"}},
+    {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg"    }},
     { "pipeline":    { "name": "common" } }
   ]
 }

salt/idh/init.sls

@@ -74,8 +74,6 @@ so-idh:
       - file: opencanary_config
     - require:
       - file: opencanary_config
-    - extra_hosts:
-      - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
 
 append_so-idh_so-status.conf:
   file.append:

salt/nginx/init.sls

@@ -95,7 +95,7 @@ so-nginx:
       - /opt/so/log/nginx/:/var/log/nginx:rw
       - /opt/so/tmp/nginx/:/var/lib/nginx:rw
       - /opt/so/tmp/nginx/:/run:rw
-      - /opt/so/saltstack/local/salt/elastic-fleet/files/so_agent-installers/:/opt/socore/html/packages
+      - /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/:/opt/socore/html/packages
   {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %}
       - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
       - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro

salt/ssl/init.sls

@@ -210,19 +210,19 @@ chownilogstashelasticfleetp8:
 # Create Symlinks to the keys so I can distribute it to all the things
 elasticfleetdircerts:
   file.directory:
-    - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs
+    - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs
     - makedirs: True
 
 efkeylink:
   file.symlink:
-    - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs/elasticfleet.p8
+    - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.p8
     - target: /etc/pki/elasticfleet.p8
     - user: socore
     - group: socore
 
 efcrtlink:
   file.symlink:
-    - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs/elasticfleet.crt
+    - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.crt
     - target: /etc/pki/elasticfleet.crt
     - user: socore
     - group: socore

salt/top.sls

@@ -59,7 +59,7 @@ base:
     {%- endif %}
     - schedule
     - docker_clean
-    - elastic-fleet.install_agent_grid
+    - elasticfleet.install_agent_grid
 
   '*_eval and G@saltversion:{{saltversion}}':
     - match: compound
@@ -147,7 +147,7 @@ base:
     - schedule
     - soctopus
     - playbook
-    - elastic-fleet
+    - elasticfleet
     - docker_clean
 
   '*_standalone and G@saltversion:{{saltversion}}':
@@ -198,7 +198,7 @@ base:
     - schedule
     - soctopus
     - playbook
-    - elastic-fleet
+    - elasticfleet
     - docker_clean
 
   '*_searchnode and G@saltversion:{{saltversion}}':
@@ -215,7 +215,7 @@ base:
     - logstash
     {%- endif %}
     - schedule
-    - elastic-fleet.install_agent_grid
+    - elasticfleet.install_agent_grid
     - docker_clean
 
   '*_managersearch and G@saltversion:{{saltversion}}':
@@ -257,7 +257,7 @@ base:
     - schedule
     - soctopus
     - playbook
-    - elastic-fleet
+    - elasticfleet
     - docker_clean
 
   '*_heavynode and G@saltversion:{{saltversion}}':
@@ -286,7 +286,7 @@ base:
     - zeek
     {%- endif %}
     - schedule
-    - elastic-fleet.install_agent_grid
+    - elasticfleet.install_agent_grid
     - docker_clean
   
   '*_import and G@saltversion:{{saltversion}}':
@@ -317,7 +317,7 @@ base:
     - suricata
     - zeek
     - schedule
-    - elastic-fleet
+    - elasticfleet
     - docker_clean
 
   '*_receiver and G@saltversion:{{saltversion}}':
@@ -333,7 +333,7 @@ base:
     - redis
     {%- endif %}
     - schedule
-    - elastic-fleet.install_agent_grid
+    - elasticfleet.install_agent_grid
     - docker_clean
 
   '*_idh and G@saltversion:{{saltversion}}':
@@ -343,7 +343,7 @@ base:
     - telegraf
     - firewall
     - schedule
-    - elastic-fleet.install_agent_grid
+    - elasticfleet.install_agent_grid
     - docker_clean
     - idh
 

setup/so-setup

@@ -58,6 +58,10 @@ while [[ $# -gt 0 ]]; do
 	esac
 done
 
+# Preserve old setup/error logs
+[ -f "$error_log" ] && mv "$error_log" "$error_log.$(date +%Y-%m-%dT%H:%M:%S)"
+[ -f "$setup_log" ] && mv "$setup_log" "$setup_log.$(date +%Y-%m-%dT%H:%M:%S)"
+
 # Let's see what OS we are dealing with here
 detect_os
 
@@ -134,9 +138,7 @@ title "Checking to see if install has run before"
 if [[ -f /root/accept_changes ]]; then
 	is_reinstall=true
 	whiptail_reinstall
-	info "Old setup detected. Moving the last setup.log to setup.log.bak"
+	info "Old setup detected. Preparing for reinstallation."
-	mv "$setup_log" "$setup_log.bak"
-	[ -f "$error_log" ] && mv "$error_log" "$error_log.bak"
 	reinstall_init
 	reset_proxy
 fi
@@ -267,7 +269,7 @@ if ! [[ -f $install_opt_file ]]; then
 	if (whiptail_you_sure); then
 		true
 	else
-		error "User cancelled setup."
+		info "User cancelled setup."
 		whiptail_cancel
 	fi
 	# If this is an analyst install lets streamline the process. 

setup/so-whiptail

@@ -976,7 +976,7 @@ whiptail_manager_unreachable() {
 
 	Run the following on the manager:
 	
-	so-firewall-minion --role=$install_type --ip=$MAINIP
+	sudo so-firewall-minion --role=$install_type --ip=$MAINIP
 	
 	Would you like to retry?
 	EOM
@@ -1271,38 +1271,54 @@ whiptail_setup_complete() {
 
 	[ -n "$TESTING" ] && return
 
-	if [[ -n "$REDIRECTIT" && $is_manager = true ]]; then
+
+	if [[ $waitforstate ]]; then
+		# Manager-type Nodes - Install Summary
 		if [[ -n $ALLOW_CIDR ]]; then
 			local sentence_prefix="Access"
 		else
 			local sentence_prefix="Run so-allow to access"
 		fi
-		local accessMessage="\n${sentence_prefix} the web interface at: https://${REDIRECTIT}\n"
+	
-	elif [[ $is_idh ]]; then
+		read -r -d '' message <<- EOM
-		local accessMessage="\nSSH for this node has been moved to TCP/2222, accessible only from the Manager node.\n"
+			${install_type} setup is now complete!
+			
+			${sentence_prefix} the Security Onion Console (SOC) web interface by navigating to:
+			https://${REDIRECTIT}
+
+			Then login with the following username and password.
+
+			SOC Username: ${WEBUSER}
+			SOC Password: Use the password that was entered during setup
+			
+			Press TAB and then the ENTER key to exit this screen.
+		EOM
+		whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext
 	else
-		local accessMessage=""
+		if [[ $is_idh ]]; then
+			local accessMessage="\nSSH for this node has been moved to TCP/2222, accessible only from the Manager node.\n"
+		else
+			local accessMessage=""
+		fi
+		MINIONFINGERPRINT=$(salt-call --local key.finger --out=newline_values_only)
+		read -r -d '' message <<- EOM
+			${install_type} initialization is now complete!
+			
+			To finish configuration, open the Security Onion Console web interface
+			and navigate to Administration -> Grid Members.
+			
+			Then find this node in the Pending Members list, 
+			click the Review button, and then click the Accept button.
+			
+			Node Hostname: $HOSTNAME
+			Node Fingerprint:
+			$MINIONFINGERPRINT
+			$accessMessage
+			Press TAB and then the ENTER key to exit this screen.
+		EOM
+
+		whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext
 	fi
-
-	MINIONFINGERPRINT=$(salt-call --local key.finger --out=newline_values_only)
-	read -r -d '' message <<- EOM
-		${install_type} initialization is now complete!
-		
-		To finish configuration, open the Security Onion Console web interface
-		and navigate to Administration -> Grid Members.
-		
-		Then find this node in the Pending Members list, 
-		click the Review button, and then click the Accept button.
-		
-		Node Hostname: $HOSTNAME
-		Node Fingerprint:
-		$MINIONFINGERPRINT
-
-		$accessMessage
-		Press TAB and then the ENTER key to exit this screen.
-	EOM
-
-	whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext
 }
 
 whiptail_setup_failed() {