Merge pull request #5142 from Security-Onion-Solutions/foxtrot

Add image pull script to allow so-learn to pull missing images, update wording on several whiptail prompts

salt/common/tools/sbin/so-image-pull

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+. /usr/sbin/so-image-common
+
+usage() {
+	read -r -d '' message <<- EOM
+		usage: so-image-pull [-h] IMAGE [IMAGE ...]
+
+		positional arguments:
+			IMAGE         One or more 'so-' prefixed images to download and verify.
+
+		optional arguments:
+			-h, --help    Show this help message and exit.
+	EOM
+	echo "$message"
+	exit 1
+}
+
+for arg; do
+	shift
+	[[ "$arg" = "--quiet" || "$arg" = "-q" ]] && quiet=true && continue
+	set -- "$@" "$arg"
+done
+
+if [[ $# -eq 0 || $# -gt 1 ]] || [[ $1 == '-h' || $1 == '--help' ]]; then
+	usage
+fi
+
+TRUSTED_CONTAINERS=("$@")
+set_version
+
+for image in "${TRUSTED_CONTAINERS[@]}"; do
+	if ! docker images | grep "$image" | grep ":5000" | grep -q "$VERSION"; then
+		if [[ $quiet == true ]]; then
+			update_docker_containers "$image" "" "" "/dev/null"
+		else
+			update_docker_containers "$image" "" "" "" 
+		fi
+	else
+		echo "$image:$VERSION image exists."
+	fi
+done

salt/common/tools/sbin/so-learn

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from itertools import chain
 from typing import List
 
 import signal
@@ -26,10 +27,12 @@ import argparse
 import textwrap
 import yaml
 import multiprocessing
+import docker
+import pty
 
 minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions'
 so_status_conf = '/opt/so/conf/so-status/so-status.conf'
-salt_proc: subprocess.CompletedProcess = None
+proc: subprocess.CompletedProcess = None
 
 # Temp store of modules, will likely be broken out into salt
 def get_learn_modules():
@@ -52,8 +55,8 @@ def get_cpu_period(fraction: float):
 
 def sigint_handler(*_):
   print('Exiting gracefully on Ctrl-C')
-  if salt_proc is not None: salt_proc.send_signal(signal.SIGINT)
-  sys.exit(0)
+  if proc is not None: proc.send_signal(signal.SIGINT)
+  sys.exit(1)
 
 
 def find_minion_pillar() -> str:
@@ -112,7 +115,6 @@ def mod_so_status(action: str, item: str):
       if action == 'remove': pass
       if action == 'add': containers.append(f'so-{item}\n')
       
-
     [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines
     
     conf.seek(0)
@@ -132,15 +134,48 @@ def create_pillar_if_not_exist(pillar:str, content: dict):
   return content
 
 
+def salt_call(module: str):
+  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True']
+
+  print(f'  Applying salt state for {module} module...')
+  proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+  return_code = proc.returncode
+  if return_code != 0:
+    print(f'  [ERROR] Failed to apply salt state for {module} module.')
+
+  return return_code
+
+
+def pull_image(module: str):
+  container_basename = f'so-{module}'
+
+  client = docker.from_env()
+  image_list = client.images.list(filters={ 'dangling': False })  
+  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
+  basename_match = list(filter(lambda x: f'{container_basename}' in x, tag_list))
+  local_registry_match = list(filter(lambda x: ':5000' in x, basename_match))
+
+  if len(local_registry_match) == 0:
+    print(f'Pulling and verifying missing image for {module} (may take several minutes) ...')
+    pull_command = ['so-image-pull', '--quiet', container_basename]
+  
+    proc = subprocess.run(pull_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    return_code = proc.returncode
+    if return_code != 0:
+      print(f'[ERROR] Failed to pull image so-{module}, skipping state.')
+  else:
+    return_code = 0
+  return return_code
+
+
 def apply(module_list: List):
   return_code = 0
   for module in module_list:
-    salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True']
-    print(f'  Applying salt state for {module} module...')
-    salt_proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
-    if salt_proc.returncode != 0:
-      print(f'  [ERROR] Failed to apply salt state for {module} module.')
-      return_code = salt_proc.returncode
+    salt_ret = salt_call(module)
+    # Only update return_code if the command returned a non-zero return
+    if salt_ret != 0:
+      return_code = salt_ret
+    
   return return_code
 
 
@@ -170,6 +205,7 @@ def enable_disable_modules(args, enable: bool):
     for module, details in pillar_modules.items():
       details['enabled'] = enable
       mod_so_status(action_str, module)
+      if enable: pull_image(module)
     args.pillar_dict.update()
     write_pillar(args.pillar, args.pillar_dict)
   else:
@@ -180,6 +216,8 @@ def enable_disable_modules(args, enable: bool):
           state_str = 'enabled' if enable else 'disabled'
           print(f'{module} module already {state_str}.', file=sys.stderr)
         else:
+          if enable and pull_image(module) != 0:
+            continue
           pillar_modules[module]['enabled'] = enable
           mod_so_status(action_str, module)
           write_needed = True

setup/so-whiptail

@@ -959,33 +959,18 @@ whiptail_management_interface_gateway() {
 whiptail_management_interface_ip_mask() {
 	[ -n "$TESTING" ] && return
 
-	manager_ip_mask=$(whiptail --title "$whiptail_title" --inputbox \
-		"Enter your IPv4 address with CIDR mask (e.g. 192.168.1.2/24):" 10 60 "$1" 3>&1 1>&2 2>&3)
+	local msg
+	read -r -d '' msg <<- EOM
+	What IPv4 address would you like to assign to this Security Onion installation?
+
+	Please enter the IPv4 address with CIDR mask
+	(e.g. 192.168.1.2/24):
+	EOM
+
+	manager_ip_mask=$(whiptail --title "$whiptail_title" --inputbox "$msg" 12 60 "$1" 3>&1 1>&2 2>&3)
 
 	local exitstatus=$?
-	whiptail_check_exitstatus $exitstatus
-}
-
-whiptail_management_interface_ip() {
-
-	[ -n "$TESTING" ] && return
-
-	MIP=$(whiptail --title "$whiptail_title" --inputbox \
-	"Enter your IP address:" 10 60 X.X.X.X 3>&1 1>&2 2>&3)
-
-	local exitstatus=$?
-	whiptail_check_exitstatus $exitstatus
-}
-
-whiptail_management_interface_mask() {
-
-	[ -n "$TESTING" ] && return
-
-	MMASK=$(whiptail --title "$whiptail_title" --inputbox \
-	"Enter the bit mask for your subnet:" 10 60 24 3>&1 1>&2 2>&3)
-
-	local exitstatus=$?
-	whiptail_check_exitstatus $exitstatus
+	# whiptail_check_exitstatus $exitstatus
 }
 
 whiptail_management_nic() {
@@ -1734,7 +1719,7 @@ whiptail_so_allow_yesno() {
 	[ -n "$TESTING" ] && return
 
 	whiptail --title "$whiptail_title" \
-		--yesno "Do you want to run so-allow to allow access to the web tools?" \
+		--yesno "Do you want to run so-allow to allow other machines to access this Security Onion installation via the web interface?" \
 		8 75
 }