Merge pull request #9643 from Security-Onion-Solutions/2.4/dev

Merge Dev

salt/common/packages.sls

@@ -1,6 +1,33 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% if GLOBALS.os != 'CentOS' %}     
+{% if GLOBALS.os == 'Ubuntu' %}
+commonpkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - chrony
+      - apache2-utils
+      - wget
+      - ntpdate
+      - jq
+      - python3-docker
+      - curl
+      - ca-certificates
+      - software-properties-common
+      - apt-transport-https
+      - openssl
+      - netcat
+      - python3-mysqldb
+      - sqlite3
+      - libssl-dev
+      - python3-dateutil
+      - python3-m2crypto
+      - python3-mysqldb
+      - python3-packaging
+      - python3-lxml
+      - git
+      - vim
+{% elif GLOBALS.os == 'Rocky' %}     
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True

salt/docker/init.sls

@@ -4,12 +4,25 @@
 # Elastic License 2.0.
 
 {% from 'docker/docker.map.jinja' import DOCKER %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
 
 dockergroup:
   group.present:
     - name: docker
     - gid: 920
 
+{% if GLOBALS.os == 'Ubuntu' %}
+dockerheldpackages:
+  pkg.installed:
+    - pkgs:
+      - containerd.io: 1.4.9-1
+      - docker-ce: 5:20.10.8~3-0~ubuntu-focal
+      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal
+      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal
+    - hold: True
+    - update_holds: True
+{% else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
@@ -19,6 +32,7 @@ dockerheldpackages:
       - docker-ce-rootless-extras: 20.10.5-3.el7
     - hold: True
     - update_holds: True
+{% endif %}
 
 #disable docker from managing iptables
 iptables_disabled:

salt/repo/client/ubuntu.sls

@@ -0,0 +1,20 @@
+# this removes the repo file left by bootstrap-salt.sh without -r
+remove_salt.list:
+  file.absent:
+    - name: /etc/apt/sources.list.d/salt.list
+
+saltstack.list:
+  file.managed:
+    - name: /etc/apt/sources.list.d/saltstack.list
+    - contents:
+      - deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/{{grains.osrelease}}/amd64/salt3004.2/ {{grains.oscodename}} main
+
+apt_update:
+  cmd.run:
+    - name: apt-get update
+    - onchanges:
+      - file: saltstack.list
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30

salt/soc/defaults.yaml

@@ -1020,6 +1020,9 @@ soc:
         cacheMs: 300000
         verifyCert: false
         casesEnabled: true
+        extractCommonObservables:
+          - source.ip
+          - destination.ip
         timeoutMs: 300000
         timeShiftMs: 120000
         defaultDurationMs: 1800000

salt/soc/init.sls

@@ -115,7 +115,7 @@ so-soc:
       {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
       - {{ SN.split('_')|first }}:{{ SNDATA.ip }}
       {%- endfor %}
-      {%- endif %}
+    {%- endif %}
     - port_bindings:
       {% for BINDING in DOCKER.containers['so-soc'].port_bindings %}
       - {{ BINDING }}

salt/soc/soc_soc.yaml

@@ -69,6 +69,9 @@ soc:
           description: Set to true if the SOC case management module, natively integrated with Elasticsearch, should be enabled.
           global: True
           advanced: True
+        extractCommonObservables:
+          description: List of indexed fields to automatically extract into a case observable, when attaching related events to a case.
+          global: True
         timeShiftMs:
           description: Duration (in milliseconds) to further expand the PCAP time range when querying PCAP data related to an event. This duration is added to the normal duration value (see defaultDurationMs).
           global: True

setup/so-functions

@@ -909,10 +909,9 @@ detect_os() {
 
 	elif [ -f /etc/os-release ]; then
 		OS=ubuntu
-		if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
-			OSVER=bionic
-		elif grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
+		if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
 			OSVER=focal
+			is_ubuntu=true
 		else
 			info "We do not support your current version of Ubuntu."
 			exit 1
@@ -1992,6 +1991,43 @@ repo_sync_local() {
 }
 
 saltify() {
+	if [[ $is_ubuntu ]]; then
+
+		DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1
+		update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 
+		local pkg_arr=(
+			'apache2-utils'
+			'ca-certificates'
+			'curl'
+			'software-properties-common'
+			'apt-transport-https'
+			'openssl'
+			'netcat'
+			'jq'
+		)
+		retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1
+
+		logCmd "mkdir -vp /opt/so/gpg"
+		logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub"
+		logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg"
+		logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub"
+		logCmd "apt-key add /opt/so/gpg/docker.pub"
+
+		# Add SO Saltstack Repo
+		echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list
+
+		# Add Docker Repo
+		add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 
+		
+		# Ain't nothing but a GPG 
+
+		retry 150 20 "apt-get update" "" "Err:" || exit 1
+		retry 150 20 "apt-get -y install salt-minion=3004.2+ds-1 salt-common=3004.2+ds-1" || exit 1
+		retry 150 20 "apt-mark hold salt-minion salt-common" || exit 1
+		retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" || exit 1
+
+	fi
+		
 	if [[ $is_centos ]]; then
 		RUSALTY=$(rpm -qa | grep salt-minion | wc -l)
 		if [[ "$RUSALTY" -gt 0 ]]; then
@@ -2305,6 +2341,22 @@ so_add_user() {
 	fi
 }
 
+ubuntu_check() {
+	if [[ $OS == "ubuntu" ]]; then
+		if [[ $waitforstate ]]; then
+			whiptail_ubuntu_notsupported
+			exit 1
+		else
+		    if [[ $UBUNTUINSTALL == "needtoupgrade" ]]; then
+				whiptail_ubuntu_warning
+			else
+				whiptail_ubuntu_notsupported
+				exit 1
+			fi
+		fi
+	fi
+}
+
 update_sudoers_for_testing() {
 	if [ -n "$TESTING" ]; then
 		info "Ensuring $INSTALLUSERNAME has password-less sudo access for automated testing purposes."
@@ -2383,9 +2435,10 @@ wait_for_salt_minion() {
 
 verify_setup() {
 	info "Verifying setup"
-	output=$(./so-verify "$setup_type" 2>&1)
+	set -o pipefail
+	./so-verify "$setup_type" 2>&1 | tee -a $setup_log
 	result=$?
-	echo "$output" >> "$setup_log"
+	set +o pipefail
 	if [[ $result -eq 0 ]]; then
 		whiptail_setup_complete
 	else

setup/so-setup

@@ -314,6 +314,7 @@ if ! [[ -f $install_opt_file ]]; then
 	# If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles.
 	if [[ $is_eval ]]; then
 		waitforstate=true
+		ubuntu_check
 		monints=true
 		check_elastic_license
 		check_requirements "manager"
@@ -336,6 +337,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 	elif [[ $is_standalone ]]; then
 		waitforstate=true
+		ubuntu_check
 		monints=true
 		check_elastic_license
 		check_requirements "manager"
@@ -359,6 +361,7 @@ if ! [[ -f $install_opt_file ]]; then
 	elif [[ $is_manager ]]; then
 		check_elastic_license
 		waitforstate=true
+		ubuntu_check
 		check_requirements "manager"
 		networking_needful
 		collect_dockernet
@@ -378,6 +381,7 @@ if ! [[ -f $install_opt_file ]]; then
 	elif [[ $is_managersearch ]]; then
 		check_elastic_license
 		waitforstate=true
+		ubuntu_check
 		check_requirements "manager"
 		networking_needful
 		collect_dockernet
@@ -395,6 +399,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_so_allow
 		whiptail_end_settings
 	elif [[ $is_sensor ]]; then
+		ubuntu_check
 		monints=true
 		check_requirements "sensor"
 		calculate_useable_cores
@@ -410,6 +415,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 		
 	elif [[ $is_searchnode ]]; then
+		ubuntu_check
 		check_requirements "elasticsearch"
 		networking_needful
 		check_network_manager_conf
@@ -422,6 +428,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_heavynode ]]; then
+		ubuntu_check
 		monints=true
 		check_requirements "heavynode"
 		calculate_useable_cores
@@ -432,6 +439,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_idh ]]; then
+		ubuntu_check
 		check_requirements "idh"
 		networking_needful
 		collect_mngr_hostname
@@ -440,6 +448,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_import ]]; then
+		ubuntu_check
 		waitforstate=true
 		monints=true
 		check_elastic_license
@@ -461,6 +470,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_receiver ]]; then
+		ubuntu_check
 		check_requirements "receiver"
 		networking_needful
 		collect_mngr_hostname

setup/so-whiptail

@@ -1327,6 +1327,28 @@ whiptail_storage_requirements() {
 	whiptail_check_exitstatus $exitstatus
 }
 
+whiptail_ubuntu_notsupported() {
+	[ -n "$TESTING" ] && return
+	
+	read -r -d '' message <<- EOM
+		Ubuntu is not supported for this node type.
+
+		Please use a supported OS or install via ISO.
+	EOM
+	whiptail --title "$whiptail_title"  --msgbox "$message" 14 75
+}
+
+whiptail_ubuntu_warning() {
+	[ -n "$TESTING" ] && return
+	
+	read -r -d '' message <<- EOM
+		Ubuntu support for this node type is limited.
+
+		Please consider using a fully supported OS or install via ISO.
+	EOM
+	whiptail --title "$whiptail_title"  --msgbox "$message" 14 75
+
+}
 whiptail_uppercase_warning() {
 
 	[ -n "$TESTING" ] && return