From d205fff3ba13d3662d2adac1485f95c354897eb9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 19 Feb 2021 13:45:23 -0500 Subject: [PATCH 01/65] Run ssh-harden in setup per #1932 --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 6eb2bc1ed..3245aacb5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1381,6 +1381,8 @@ install_cleanup() { info "Removing so-setup permission entry from sudoers file" sed -i '/so-setup/d' /etc/sudoers fi + + so-ssh-harden -q } import_registry_docker() { From 33696398eb877e8a668efcd3e08f48c5b8b6b1a1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 26 Feb 2021 18:06:07 -0500 Subject: [PATCH 02/65] Add new so-docker-prune script * Script will pull list of so- images and prune any older than most recent + last version --- salt/common/tools/sbin/so-docker-prune | 77 +++++++++++++++++ salt/common/tools/sbin/so-monitor-add | 0 .../tools/sbin/so-playbook-sigma-refresh | 0 salt/common/tools/sbin/so-raid-status | 0 salt/common/tools/sbin/so-rule | 0 salt/common/tools/sbin/so-suricata-testrule | 0 salt/docker_clean/init.sls | 83 +------------------ setup/so-functions | 5 +- 8 files changed, 83 insertions(+), 82 deletions(-) create mode 100755 salt/common/tools/sbin/so-docker-prune mode change 100644 => 100755 salt/common/tools/sbin/so-monitor-add mode change 100644 => 100755 salt/common/tools/sbin/so-playbook-sigma-refresh mode change 100644 => 100755 salt/common/tools/sbin/so-raid-status mode change 100644 => 100755 salt/common/tools/sbin/so-rule mode change 100644 => 100755 salt/common/tools/sbin/so-suricata-testrule diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune new file mode 100755 index 000000000..1b6242488 --- /dev/null +++ b/salt/common/tools/sbin/so-docker-prune @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys, argparse, re, docker +from packaging.version import Version +from itertools import groupby, chain + + +def get_image_name(string) -> str: + return ':'.join(string.split(':')[:-1]) + + +def get_so_image_basename(string) -> str: + return get_image_name(string).split('/so-')[-1] + + +def get_image_version(string) -> str: + ver = string.split(':')[-1] + if ver == 'latest': + # Version doesn't like "latest", so use a high semver + return '999999.9.9' + else: + return ver + + +def main(quiet): + client = docker.from_env() + + # Get list of non-dangling images + image_list = client.images.list(filters={ 'dangling': False }) + + # Map image objects to flattened list of tags (format: "name:version") + tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list)))) + + # Filter to only SO images (base name begins with "so-") + tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list)) + + # Group tags into lists by base name (sort by same projection first) + tag_list.sort(key=lambda x: get_so_image_basename(x)) + grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ] + + no_prunable = True + for t_list in grouped_tag_lists: + # Keep the 2 most current images + t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True) + if len(t_list) <= 2: + continue + else: + no_prunable = False + for tag in t_list[2:]: + if not quiet: print(f'Removing image {tag}') + client.images.remove(tag) + + if no_prunable and not quiet: + print('No Security Onion images to prune') + + +if __name__ == "__main__": + main_parser = argparse.ArgumentParser(add_help=False) + main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False) + args = main_parser.parse_args(sys.argv[1:]) + + main(args.quiet) \ No newline at end of file diff --git a/salt/common/tools/sbin/so-monitor-add b/salt/common/tools/sbin/so-monitor-add old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-playbook-sigma-refresh b/salt/common/tools/sbin/so-playbook-sigma-refresh old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-suricata-testrule b/salt/common/tools/sbin/so-suricata-testrule old mode 100644 new mode 100755 diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index 006d108eb..a92d3aedd 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -1,86 +1,9 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} -{% set MANAGER = salt['grains.get']('master') %} -{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2','2.3.10','2.3.20']%} - -{% for VERSION in OLDVERSIONS %} -remove_images_{{ VERSION }}: - docker_image.absent: - - force: True - - images: - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-acng:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-cortex:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elastalert:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet-launcher:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-grafana:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-influxdb:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kratos:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-mysql:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nginx:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-redis:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soc:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soctopus:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-steno:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-frontend:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-manager:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-backend:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-filestream:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-telegraf:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-acng:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-thehive-cortex:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-curator:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-domainstats:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-elastalert:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-elasticsearch:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-filebeat:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-fleet:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-fleet-launcher:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-freqserver:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-grafana:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-idstools:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-influxdb:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-kibana:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-kratos:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-logstash:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-minio:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-mysql:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-nginx:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-pcaptools:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-playbook:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-redis:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-soc:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-soctopus:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-steno:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-frontend:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-manager:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-backend:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-filestream:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-suricata:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-telegraf:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-thehive:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-thehive-es:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-wazuh:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-zeek:{{ VERSION }}' -{% endfor %} +prune_images: + cmd.run: + - name: so-docker-prune {% else %} diff --git a/setup/so-functions b/setup/so-functions index 0ad993d3d..30075bb0c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1957,6 +1957,7 @@ saltify() { python36-dateutil\ python36-m2crypto\ python36-mysql\ + python36-packaging\ yum-utils\ device-mapper-persistent-data\ lvm2\ @@ -2045,9 +2046,9 @@ saltify() { retry 50 10 "apt-get -y install salt-minion=3002.2+ds-1 salt-common=3002.2+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1 else - retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-packaging" >> "$setup_log" 2>&1 || exit 1 fi fi } From 0db999130706e4c4aac729d1afebf87fcc2d0585 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 10:20:33 -0500 Subject: [PATCH 03/65] Reword/remove some comments --- salt/common/tools/sbin/so-docker-prune | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 1b6242488..06e05d7dc 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -40,10 +40,9 @@ def get_image_version(string) -> str: def main(quiet): client = docker.from_env() - # Get list of non-dangling images image_list = client.images.list(filters={ 'dangling': False }) - # Map image objects to flattened list of tags (format: "name:version") + # Map list of image objects to flattened list of tags (format: "name:version") tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list)))) # Filter to only SO images (base name begins with "so-") From 497938460a83de8d86e9b23393ba9f53ecd773ae Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 16:47:49 -0500 Subject: [PATCH 04/65] [fix] manager:log_size_limit is no longer used, remove generation --- setup/files/.curlrc | 0 setup/files/.wgetrc | 0 setup/so-functions | 1 - 3 files changed, 1 deletion(-) create mode 100644 setup/files/.curlrc create mode 100644 setup/files/.wgetrc diff --git a/setup/files/.curlrc b/setup/files/.curlrc new file mode 100644 index 000000000..e69de29bb diff --git a/setup/files/.wgetrc b/setup/files/.wgetrc new file mode 100644 index 000000000..e69de29bb diff --git a/setup/so-functions b/setup/so-functions index 21602f320..f19f60e72 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1446,7 +1446,6 @@ manager_pillar() { printf '%s\n'\ " elastalert: 1"\ " es_port: $node_es_port"\ - " log_size_limit: $log_size_limit"\ " cur_close_days: $CURCLOSEDAYS"\ " grafana: $GRAFANA"\ " osquery: $OSQUERY"\ From 4df53b3c70cebc2c8c5b0c27e62e972ad908ce48 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 17:38:17 -0500 Subject: [PATCH 05/65] Unify log_size_limit variable value in so-curator-closed-delete-delete --- salt/curator/files/bin/so-curator-closed-delete-delete | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 2a9165921..58433ee1a 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -4,12 +4,11 @@ {%- if grains['role'] in ['so-node', 'so-heavynode'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%} - {%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%} {%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('manager:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('manager:es_port', '') -%} - {%- set LOG_SIZE_LIMIT = salt['pillar.get']('manager:log_size_limit', '') -%} {%- endif -%} +{%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%} # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # From de77d3ebc99733cac0eb304507f745e4071198c9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 17:41:49 -0500 Subject: [PATCH 06/65] [wip] Initial work for setting up proxy on manager --- salt/common/tools/sbin/so-common | 20 ++++++++++++ setup/so-functions | 54 ++++++++++++++++++++++++++++++-- setup/so-variables | 2 ++ setup/so-whiptail | 15 +++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index b76ad05ff..cdc8b4772 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -340,6 +340,26 @@ valid_int() { # {% raw %} +valid_proxy() { + local proxy=$1 + local url_prefixes=( 'http://' 'https://' ) + + local has_prefix=false + for prefix in "${url_prefixes[@]}"; do + echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} + done + + local url + readarray -t url -d ':' <<< "$proxy" + + local valid_url=true + if ! valid_ip4 "$proxy" && ! valid_fqdn "$proxy"; then + valid_url=false + fi + + [[ $has_prefix == true ]] && [[ $valid_url ]] && return 0 || return 1 +} + valid_string() { local str=$1 local min_length=${2:-1} diff --git a/setup/so-functions b/setup/so-functions index f19f60e72..d457e7005 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -535,6 +535,17 @@ collect_patch_schedule_name_import() { done } +collect_proxy() { + if whiptail_proxy_ask; then + whiptail_proxy_addr + + while ! valid_proxy "$proxy_addr"; do + whiptail_invalid_input + whiptail_proxy_addr + done + fi +} + collect_redirect_host() { whiptail_set_redirect_host "$HOSTNAME" @@ -1511,7 +1522,7 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$PROXY'"\ + " proxy: '$proxy_addr'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ @@ -2184,7 +2195,46 @@ set_main_ip() { # Add /usr/sbin to everyone's path set_path() { - echo "complete -cf sudo" > /etc/profile.d/securityonion.sh + echo "complete -cf sudo" >> "$profile_d_config_file" +} + +set_proxy() { + # Don't proxy localhost, local ip, and management ip + local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}" + + # Set proxy environment variables used by curl, wget, docker, and others + { + echo "export use_proxy=on" + echo "export http_proxy=\"${proxy_addr}\"" + echo "export https_proxy=\"\$http_addr\"" + echo "export ftp_proxy=\"\$http_addr\"" + echo "export no_proxy=\"${no_proxy_string}\"" + } >> "$profile_d_config_file" + + # Create proxy config for dockerd + printf '%s\n'\ + "[Service]"\ + "Environment=\"HTTP_PROXY=${proxy_addr}\""\ + "Environment=\"HTTPS_PROXY=${proxy_addr}\""\ + "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf + + systemctl daemon-reload + systemctl restart docker + + # Set proxy for package manager + if [ "$OS" = 'centos' ]; then + echo "proxy=$proxy_addr" >> /etc/yum.conf + else + # Set it up so the updates roll through the manager + printf '%s\n'\ + "Acquire::http::Proxy \"$proxy_addr\";"\ + "Acquire::https::Proxy \"$proxy_addr\";" > /etc/apt/apt.conf.d/00-proxy.conf + fi + + # Set global git proxy + printf '%s\n'\ + "[http]"\ + " proxy = ${proxy_addr}" > /etc/gitconfig } setup_salt_master_dirs() { diff --git a/setup/so-variables b/setup/so-variables index a2fdf03c6..cd6478fa7 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -72,3 +72,5 @@ export install_opt_file net_init_file=/root/net_init export net_init_file + +export profile_d_config_file='/etc/profile.d/securityonion.sh' diff --git a/setup/so-whiptail b/setup/so-whiptail index 7bb265de1..6baa143ad 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1216,6 +1216,21 @@ whiptail_patch_schedule_select_hours() { } +whiptail_proxy_ask() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "Do you want to use a proxy server to complete setup?" 7 60 +} + +whiptail_proxy_addr() { + [ -n "$TESTING" ] && return + + proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234):" 8 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_requirements_error() { local requirement_needed=$1 From 2b94fa366ed785750382edaa57a936253d0cc34f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 09:51:38 -0500 Subject: [PATCH 07/65] [wip] Add auth inputs for proxy settings, fix some broken logic --- salt/common/tools/sbin/so-common | 4 ++-- setup/so-functions | 29 +++++++++++++++++++++++++++++ setup/so-whiptail | 18 ++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index cdc8b4772..438fe9329 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -346,7 +346,7 @@ valid_proxy() { local has_prefix=false for prefix in "${url_prefixes[@]}"; do - echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} + echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break done local url @@ -357,7 +357,7 @@ valid_proxy() { valid_url=false fi - [[ $has_prefix == true ]] && [[ $valid_url ]] && return 0 || return 1 + [[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1 } valid_string() { diff --git a/setup/so-functions b/setup/so-functions index d457e7005..c8adbc9f5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -537,12 +537,41 @@ collect_patch_schedule_name_import() { collect_proxy() { if whiptail_proxy_ask; then + whiptail_proxy_addr while ! valid_proxy "$proxy_addr"; do whiptail_invalid_input whiptail_proxy_addr done + + if whiptail_proxy_auth_ask; then + whiptail_proxy_auth_user + + while ! valid_string "$proxy_user"; do + whiptail_invalid_input + whiptail_proxy_auth_user + done + + whiptail_proxy_auth_pass + + while ! valid_string "$proxy_pass"; do + whiptail_invalid_input + whiptail_proxy_auth_pass + done + + local url_prefixes=( 'http://' 'https://' ) + for prefix in "${url_prefixes[@]}"; do + if echo "$proxy_addr" | grep "$prefix"; then + proxy=${proxy_addr#"$prefix"} + proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}" + break + fi + done + else + proxy="$proxy_addr" + fi + export proxy fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index 6baa143ad..205bf6f98 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1231,6 +1231,24 @@ whiptail_proxy_addr() { whiptail_check_exitstatus $exitstatus } +whiptail_proxy_auth_ask() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "Does your proxy require authentication?" 7 60 +} + +whiptail_proxy_auth_user() { + [ -n "$TESTING" ] && return + + proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 3>&1 1>&2 2>&3) +} + +whiptail_proxy_auth_pass() { + [ -n "$TESTING" ] && return + + proxy_pass=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) +} + whiptail_requirements_error() { local requirement_needed=$1 From 82018a206c356a3cd048960ca01ecc8d63e3cdaf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 09:56:14 -0500 Subject: [PATCH 08/65] [wip] Don't validate user+pass for proxy, use new variable --- setup/so-functions | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c8adbc9f5..1ae0d092c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -548,18 +548,8 @@ collect_proxy() { if whiptail_proxy_auth_ask; then whiptail_proxy_auth_user - while ! valid_string "$proxy_user"; do - whiptail_invalid_input - whiptail_proxy_auth_user - done - whiptail_proxy_auth_pass - while ! valid_string "$proxy_pass"; do - whiptail_invalid_input - whiptail_proxy_auth_pass - done - local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do if echo "$proxy_addr" | grep "$prefix"; then @@ -1551,7 +1541,7 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$proxy_addr'"\ + " proxy: '$proxy'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ @@ -2234,7 +2224,7 @@ set_proxy() { # Set proxy environment variables used by curl, wget, docker, and others { echo "export use_proxy=on" - echo "export http_proxy=\"${proxy_addr}\"" + echo "export http_proxy=\"${proxy}\"" echo "export https_proxy=\"\$http_addr\"" echo "export ftp_proxy=\"\$http_addr\"" echo "export no_proxy=\"${no_proxy_string}\"" @@ -2243,8 +2233,8 @@ set_proxy() { # Create proxy config for dockerd printf '%s\n'\ "[Service]"\ - "Environment=\"HTTP_PROXY=${proxy_addr}\""\ - "Environment=\"HTTPS_PROXY=${proxy_addr}\""\ + "Environment=\"HTTP_PROXY=${proxy}\""\ + "Environment=\"HTTPS_PROXY=${proxy}\""\ "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload @@ -2252,18 +2242,18 @@ set_proxy() { # Set proxy for package manager if [ "$OS" = 'centos' ]; then - echo "proxy=$proxy_addr" >> /etc/yum.conf + echo "proxy=$proxy" >> /etc/yum.conf else # Set it up so the updates roll through the manager printf '%s\n'\ - "Acquire::http::Proxy \"$proxy_addr\";"\ - "Acquire::https::Proxy \"$proxy_addr\";" > /etc/apt/apt.conf.d/00-proxy.conf + "Acquire::http::Proxy \"$proxy\";"\ + "Acquire::https::Proxy \"$proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf fi # Set global git proxy printf '%s\n'\ "[http]"\ - " proxy = ${proxy_addr}" > /etc/gitconfig + " proxy = ${proxy}" > /etc/gitconfig } setup_salt_master_dirs() { From 279a5b60b84bd9a74cad558cb4ec30e6eabc6940 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 11:58:10 -0500 Subject: [PATCH 09/65] Soup indent fixes --- salt/common/tools/sbin/soup | 213 ++++++++++++++++++------------------ 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6eb01a694..e33d0cc41 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -105,12 +105,12 @@ check_airgap() { # See if this is an airgap install AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') if [[ "$AIRGAP" == "True" ]]; then - is_airgap=0 - UPDATE_DIR=/tmp/soagupdate/SecurityOnion - AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/Packages + is_airgap=0 + UPDATE_DIR=/tmp/soagupdate/SecurityOnion + AGDOCKER=/tmp/soagupdate/docker + AGREPO=/tmp/soagupdate/Packages else - is_airgap=1 + is_airgap=1 fi } @@ -256,14 +256,14 @@ playbook() { } pillar_changes() { - # This function is to add any new pillar items if needed. - echo "Checking to see if pillar changes are needed." - - [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 - [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 - [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 - [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 - [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 + # This function is to add any new pillar items if needed. + echo "Checking to see if pillar changes are needed." + + [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 + [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 + [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 + [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 + [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 } rc1_to_rc2() { @@ -296,15 +296,14 @@ rc1_to_rc2() { done > /opt/so/saltstack/local/pillar/global.sls sed -i 's/playbook:/playbook_db:/' /opt/so/saltstack/local/pillar/secrets.sls @@ -385,7 +384,6 @@ up_2.3.0_to_2.3.20(){ fi INSTALLEDVERSION=2.3.20 - } up_2.3.2X_to_2.3.30() { @@ -395,11 +393,11 @@ up_2.3.2X_to_2.3.30() { sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar" done -# Change the IMAGEREPO + # Change the IMAGEREPO sed -i "/ imagerepo: 'securityonion'/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls sed -i "/ imagerepo: securityonion/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls -# Strelka rule repo pillar addition + # Strelka rule repo pillar addition if [ $is_airgap -eq 0 ]; then # Add manager as default Strelka YARA rule repo sed -i "/^strelka:/a \\ repos: \n - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls; @@ -414,10 +412,10 @@ space_check() { # Check to see if there is enough space CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then - echo "You are low on disk space. Upgrade will try and clean up space."; - clean_dockers + echo "You are low on disk space. Upgrade will try and clean up space."; + clean_dockers else - echo "Plenty of space for upgrading" + echo "Plenty of space for upgrading" fi } @@ -427,16 +425,16 @@ thehive_maint() { COUNT=0 THEHIVE_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert" - if [ $? -eq 0 ]; then - THEHIVE_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi + curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert" + if [ $? -eq 0 ]; then + THEHIVE_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi done if [ "$THEHIVE_CONNECTED" == "yes" ]; then echo "Migrating thehive databases if needed." @@ -471,80 +469,80 @@ update_version() { } upgrade_check() { - # Let's make sure we actually need to update. - NEWVERSION=$(cat $UPDATE_DIR/VERSION) - if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then - echo "You are already running the latest version of Security Onion." - exit 0 - fi + # Let's make sure we actually need to update. + NEWVERSION=$(cat $UPDATE_DIR/VERSION) + if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then + echo "You are already running the latest version of Security Onion." + exit 0 + fi } upgrade_check_salt() { - NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'}) - if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then - echo "You are already running the correct version of Salt for Security Onion." - else - UPGRADESALT=1 - fi + NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'}) + if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then + echo "You are already running the correct version of Salt for Security Onion." + else + UPGRADESALT=1 + fi } upgrade_salt() { - SALTUPGRADED=True - echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." - echo "" - # If CentOS - if [ "$OS" == "centos" ]; then - echo "Removing yum versionlock for Salt." - echo "" - yum versionlock delete "salt-*" - echo "Updating Salt packages and restarting services." - echo "" - if [ $is_airgap -eq 0 ]; then - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" - else - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - fi - echo "Applying yum versionlock for Salt." - echo "" - yum versionlock add "salt-*" - # Else do Ubuntu things - elif [ "$OS" == "ubuntu" ]; then - echo "Removing apt hold for Salt." - echo "" - apt-mark unhold "salt-common" - apt-mark unhold "salt-master" - apt-mark unhold "salt-minion" - echo "Updating Salt packages and restarting services." - echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - echo "Applying apt hold for Salt." - echo "" - apt-mark hold "salt-common" - apt-mark hold "salt-master" - apt-mark hold "salt-minion" - fi + SALTUPGRADED=True + echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." + echo "" + # If CentOS + if [ "$OS" == "centos" ]; then + echo "Removing yum versionlock for Salt." + echo "" + yum versionlock delete "salt-*" + echo "Updating Salt packages and restarting services." + echo "" + if [ $is_airgap -eq 0 ]; then + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + else + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + fi + echo "Applying yum versionlock for Salt." + echo "" + yum versionlock add "salt-*" + # Else do Ubuntu things + elif [ "$OS" == "ubuntu" ]; then + echo "Removing apt hold for Salt." + echo "" + apt-mark unhold "salt-common" + apt-mark unhold "salt-master" + apt-mark unhold "salt-minion" + echo "Updating Salt packages and restarting services." + echo "" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + echo "Applying apt hold for Salt." + echo "" + apt-mark hold "salt-common" + apt-mark hold "salt-master" + apt-mark hold "salt-minion" + fi } verify_latest_update_script() { - # Check to see if the update scripts match. If not run the new one. - CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}') - GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}') - CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}') - GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') - CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}') - GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') + # Check to see if the update scripts match. If not run the new one. + CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}') + GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}') + CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}') + GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') + CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}') + GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') - if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then - echo "This version of the soup script is up to date. Proceeding." - else - echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" - cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - salt-call state.apply common queue=True - echo "" - echo "soup has been updated. Please run soup again." - exit 0 - fi + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then + echo "This version of the soup script is up to date. Proceeding." + else + echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" + cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + salt-call state.apply common queue=True + echo "" + echo "soup has been updated. Please run soup again." + exit 0 + fi } main () { @@ -557,9 +555,10 @@ while getopts ":b" opt; do echo "Batch size must be a number greater than 0." exit 1 fi - ;; - \? ) echo "Usage: cmd [-b]" - ;; + ;; + \? ) + echo "Usage: cmd [-b]" + ;; esac done From 83791d87c7a46053939199fd5ecbe787bb4a0d72 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 11:58:45 -0500 Subject: [PATCH 10/65] [wip][fix] Use passwordbox for proxy password --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 205bf6f98..cc08acf01 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1246,7 +1246,7 @@ whiptail_proxy_auth_user() { whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - proxy_pass=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) } whiptail_requirements_error() { From 26b1da744c08bfe2a9bbb1d1a7bd47ff9bb74d62 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 12:01:15 -0500 Subject: [PATCH 11/65] [wip] Reword proxy yesno prompt --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index cc08acf01..2f2ff2f78 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1219,7 +1219,7 @@ whiptail_patch_schedule_select_hours() { whiptail_proxy_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Do you want to use a proxy server to complete setup?" 7 60 + whiptail --title "Security Onion Setup" --yesno "Do you want to set a proxy server for this installation?" 7 60 } whiptail_proxy_addr() { From e951e9d9c5aff18b00d63767494d093b1247daf2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 12:19:14 -0500 Subject: [PATCH 12/65] [wip] Further proxy changes * Remove unused docker.conf template * Rename proxy variable to avoid name collision * Reword address prompt to specify users should not include user:pass in their input * Actually call the collect_proxy function --- setup/proxies/docker.conf | 2 -- setup/so-functions | 27 ++++++++++++++++----------- setup/so-whiptail | 8 +++++++- 3 files changed, 23 insertions(+), 14 deletions(-) delete mode 100644 setup/proxies/docker.conf diff --git a/setup/proxies/docker.conf b/setup/proxies/docker.conf deleted file mode 100644 index 9ab2c4b4c..000000000 --- a/setup/proxies/docker.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -ExecStart=/usr/bin/dockerd /usr/bin/dockerd -H fd:// --registry-mirror "$proxy_addr" diff --git a/setup/so-functions b/setup/so-functions index 1ae0d092c..f0b139f6d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -553,13 +553,13 @@ collect_proxy() { local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do if echo "$proxy_addr" | grep "$prefix"; then - proxy=${proxy_addr#"$prefix"} - proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}" + local proxy=${proxy_addr#"$prefix"} + so_proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}" break fi done else - proxy="$proxy_addr" + so_proxy="$proxy_addr" fi export proxy fi @@ -1541,7 +1541,7 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$proxy'"\ + " proxy: '$so_proxy'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ @@ -1719,6 +1719,10 @@ network_init() { if [[ "$setup_type" == 'iso' ]]; then set_management_interface fi + # Check if the proxy variable was set then set proxy for various apps if so + if [[ -n "$so_proxy" ]]; then + set_proxy + fi } network_init_whiptail() { @@ -1742,6 +1746,7 @@ network_init_whiptail() { whiptail_management_nic ;; esac + collect_proxy } network_setup() { @@ -2224,7 +2229,7 @@ set_proxy() { # Set proxy environment variables used by curl, wget, docker, and others { echo "export use_proxy=on" - echo "export http_proxy=\"${proxy}\"" + echo "export http_proxy=\"${so_proxy}\"" echo "export https_proxy=\"\$http_addr\"" echo "export ftp_proxy=\"\$http_addr\"" echo "export no_proxy=\"${no_proxy_string}\"" @@ -2233,8 +2238,8 @@ set_proxy() { # Create proxy config for dockerd printf '%s\n'\ "[Service]"\ - "Environment=\"HTTP_PROXY=${proxy}\""\ - "Environment=\"HTTPS_PROXY=${proxy}\""\ + "Environment=\"HTTP_PROXY=${so_proxy}\""\ + "Environment=\"HTTPS_PROXY=${so_proxy}\""\ "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload @@ -2242,18 +2247,18 @@ set_proxy() { # Set proxy for package manager if [ "$OS" = 'centos' ]; then - echo "proxy=$proxy" >> /etc/yum.conf + echo "proxy=$so_proxy" >> /etc/yum.conf else # Set it up so the updates roll through the manager printf '%s\n'\ - "Acquire::http::Proxy \"$proxy\";"\ - "Acquire::https::Proxy \"$proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf + "Acquire::http::Proxy \"$so_proxy\";"\ + "Acquire::https::Proxy \"$so_proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf fi # Set global git proxy printf '%s\n'\ "[http]"\ - " proxy = ${proxy}" > /etc/gitconfig + " proxy = ${so_proxy}" > /etc/gitconfig } setup_salt_master_dirs() { diff --git a/setup/so-whiptail b/setup/so-whiptail index 2f2ff2f78..ffa03e128 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1224,8 +1224,14 @@ whiptail_proxy_ask() { whiptail_proxy_addr() { [ -n "$TESTING" ] && return + local message + read -r -d '' message <<- EOM + Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234). - proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234):" 8 60 3>&1 1>&2 2>&3) + If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. + EOM + + proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 4ffa0fbc131393b790ec0669fd9d07c48ddff0bc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 14:09:59 -0500 Subject: [PATCH 13/65] [wip] Fix proxy validation --- salt/common/tools/sbin/so-common | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 438fe9329..25dfc2fab 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -346,14 +346,14 @@ valid_proxy() { local has_prefix=false for prefix in "${url_prefixes[@]}"; do - echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break + echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break done - local url - readarray -t url -d ':' <<< "$proxy" + local url_arr + mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")" local valid_url=true - if ! valid_ip4 "$proxy" && ! valid_fqdn "$proxy"; then + if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}"; then valid_url=false fi From 1ea3cb1c615fc876b1429ada48bca2c81f62ecd7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 14:20:26 -0500 Subject: [PATCH 14/65] [wip] Change when proxy is set up so main ip is known * Also only restart docker if the command exists (i.e. docker is installed) --- setup/so-functions | 12 ++++++++---- setup/so-setup | 4 +--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index f0b139f6d..d61470d30 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -774,10 +774,10 @@ compare_main_nic_ip() { if ! [[ $MNIC =~ ^(tun|wg|vpn).*$ ]]; then if [[ "$MAINIP" != "$MNIC_IP" ]]; then read -r -d '' message <<- EOM - The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). + The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). This is not a supported configuration, please remediate and rerun setup. - EOM + EOM whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi @@ -1719,9 +1719,11 @@ network_init() { if [[ "$setup_type" == 'iso' ]]; then set_management_interface fi + set_main_ip >> $setup_log 2>&1 + compare_main_nic_ip # Check if the proxy variable was set then set proxy for various apps if so if [[ -n "$so_proxy" ]]; then - set_proxy + set_proxy >> $setup_log 2>&1 fi } @@ -2235,6 +2237,8 @@ set_proxy() { echo "export no_proxy=\"${no_proxy_string}\"" } >> "$profile_d_config_file" + [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' + # Create proxy config for dockerd printf '%s\n'\ "[Service]"\ @@ -2243,7 +2247,7 @@ set_proxy() { "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload - systemctl restart docker + command -v docker &> /dev/null && systemctl restart docker # Set proxy for package manager if [ "$OS" = 'centos' ]; then diff --git a/setup/so-setup b/setup/so-setup index 65fbbe16e..f16699e58 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -537,9 +537,7 @@ catch() { exit } -# This block sets REDIRECTIT which is used by a function outside the below subshell -set_main_ip >> $setup_log 2>&1 -compare_main_nic_ip +# Set REDIRECTIT variable, which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 # Begin install From 3d5cf128ae579e35920f271926c4057fa5ac2ba9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 15:02:21 -0500 Subject: [PATCH 15/65] [wip] Test proxy before using it --- setup/so-functions | 18 ++++++++++++++++-- setup/so-setup | 17 +++++++++++++++++ setup/so-whiptail | 15 ++++++++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d61470d30..861dd2e14 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -536,7 +536,8 @@ collect_patch_schedule_name_import() { } collect_proxy() { - if whiptail_proxy_ask; then + local ask=${1:-true} + if [[ $ask == true ]] && whiptail_proxy_ask; then whiptail_proxy_addr @@ -1748,7 +1749,6 @@ network_init_whiptail() { whiptail_management_nic ;; esac - collect_proxy } network_setup() { @@ -1813,6 +1813,17 @@ print_salt_state_apply() { echo "Applying $state Salt state" } +proxy_validate() { + local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" + curl "$test_url" --proxy "$so_proxy" &> /dev/null + local ret=$? + + if [[ $ret != 0 ]]; then + error "Could not reach $test_url using proxy $so_proxy" + fi + return $ret +} + reserve_group_ids() { # This is a hack to fix CentOS from taking group IDs that we need groupadd -g 928 kratos @@ -2225,6 +2236,7 @@ set_path() { } set_proxy() { + # Don't proxy localhost, local ip, and management ip local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}" @@ -2237,6 +2249,8 @@ set_proxy() { echo "export no_proxy=\"${no_proxy_string}\"" } >> "$profile_d_config_file" + source "$profile_d_config_file" + [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' # Create proxy config for dockerd diff --git a/setup/so-setup b/setup/so-setup index f16699e58..18bec9983 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -198,6 +198,14 @@ if ! [[ -f $install_opt_file ]]; then printf '%s\n' \ "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" > "$net_init_file" + if [[ $is_manager ]]; then + collect_proxy + while ! proxy_validate; do + whiptail_invalid_proxy + collect_proxy no_ask + done + [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + fi whiptail_net_setup_complete else whiptail_install_type @@ -289,6 +297,15 @@ if ! [[ -f $install_opt_file ]]; then if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init fi + + if [[ $is_manager ]]; then + collect_proxy + while ! proxy_validate; do + whiptail_invalid_proxy + collect_proxy no_ask + done + [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + fi if [[ -n "$TURBO" ]]; then use_turbo_proxy diff --git a/setup/so-whiptail b/setup/so-whiptail index ffa03e128..56c0ecc5f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -588,8 +588,21 @@ whiptail_invalid_input() { # TODO: This should accept a list of arguments to spe } +whiptail_invalid_proxy() { + [ -n "$TESTING" ] && return + + local message + read -r -d '' message <<- EOM + Could not reach test url using proxy ${so_proxy}. + + Check log for details. + EOM + + whiptail --title "Security Onion Setup" --msgbox "$message" 10 60 +} + whiptail_invalid_string() { - [ -n "$TESTING" ] && return + [ -n "$TESTING" ] && return whiptail --title "Security Onion Setup" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 From 7ee0fd63752a551139f84a6dc20082a8f199f42a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:31:22 -0500 Subject: [PATCH 16/65] [wip] Specify setup log location to user when directing them to it --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 56c0ecc5f..eaa8e8af1 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -595,7 +595,7 @@ whiptail_invalid_proxy() { read -r -d '' message <<- EOM Could not reach test url using proxy ${so_proxy}. - Check log for details. + Check log (${setup_log}) for details. EOM whiptail --title "Security Onion Setup" --msgbox "$message" 10 60 From 5a97341d332f222407b63d07dc98ca44339f17f2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:41:36 -0500 Subject: [PATCH 17/65] [wip] Fix how collect_proxy function works on retry --- setup/so-functions | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 861dd2e14..2b757bad6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -537,7 +537,15 @@ collect_patch_schedule_name_import() { collect_proxy() { local ask=${1:-true} - if [[ $ask == true ]] && whiptail_proxy_ask; then + local use_proxy + if [[ $ask != true ]]; then + use_proxy=0 + else + whiptail_proxy_ask + use_proxy=$? + fi + + if [[ $use_proxy == 0 ]]; then whiptail_proxy_addr From 13dc8221978cc8b7e9811d938d2b216d0af486af Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:53:08 -0500 Subject: [PATCH 18/65] [wip] Ask user if they want to re-enter the proxy --- setup/so-functions | 5 ++--- setup/so-setup | 8 ++++++-- setup/so-whiptail | 8 ++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 2b757bad6..a3db35e41 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -551,12 +551,11 @@ collect_proxy() { while ! valid_proxy "$proxy_addr"; do whiptail_invalid_input - whiptail_proxy_addr + whiptail_proxy_addr "$proxy_addr" done if whiptail_proxy_auth_ask; then - whiptail_proxy_auth_user - + whiptail_proxy_auth_user "$proxy_user" whiptail_proxy_auth_pass local url_prefixes=( 'http://' 'https://' ) diff --git a/setup/so-setup b/setup/so-setup index 18bec9983..66542c6f3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -301,8 +301,12 @@ if ! [[ -f $install_opt_file ]]; then if [[ $is_manager ]]; then collect_proxy while ! proxy_validate; do - whiptail_invalid_proxy - collect_proxy no_ask + if whiptail_invalid_proxy; then + collect_proxy no_ask + else + proxy_addr="" + break + fi done [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index eaa8e8af1..b7f6fad6c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -593,12 +593,12 @@ whiptail_invalid_proxy() { local message read -r -d '' message <<- EOM - Could not reach test url using proxy ${so_proxy}. + Could not reach test url using proxy ${proxy_addr}. Check log (${setup_log}) for details. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 10 60 + whiptail --title "Security Onion Setup" --yesno "$message" --yestext "Enter Again" --notext "Skip" 10 60 } whiptail_invalid_string() { @@ -1244,7 +1244,7 @@ whiptail_proxy_addr() { If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. EOM - proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 3>&1 1>&2 2>&3) + proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1259,7 +1259,7 @@ whiptail_proxy_auth_ask() { whiptail_proxy_auth_user() { [ -n "$TESTING" ] && return - proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 3>&1 1>&2 2>&3) + proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) } whiptail_proxy_auth_pass() { From 2e56252f54ed327407b6ca88ff0179da6b733758 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:54:21 -0500 Subject: [PATCH 19/65] [wip] Syntax fixes --- setup/so-functions | 2 +- setup/so-whiptail | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a3db35e41..426d34b0d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -556,7 +556,7 @@ collect_proxy() { if whiptail_proxy_auth_ask; then whiptail_proxy_auth_user "$proxy_user" - whiptail_proxy_auth_pass + whiptail_proxy_auth_pass # Don't pass in existing pass since it's obfuscated local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do diff --git a/setup/so-whiptail b/setup/so-whiptail index b7f6fad6c..f62d4c32b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -598,7 +598,7 @@ whiptail_invalid_proxy() { Check log (${setup_log}) for details. EOM - whiptail --title "Security Onion Setup" --yesno "$message" --yestext "Enter Again" --notext "Skip" 10 60 + whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 10 60 } whiptail_invalid_string() { From d19ca943ccaacbe204847d5fdcb073e0441df68c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:57:16 -0500 Subject: [PATCH 20/65] [fix][wip] Only setup proxy early on configure network setup --- setup/so-functions | 4 ---- setup/so-setup | 9 +++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 426d34b0d..e082852e6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1729,10 +1729,6 @@ network_init() { fi set_main_ip >> $setup_log 2>&1 compare_main_nic_ip - # Check if the proxy variable was set then set proxy for various apps if so - if [[ -n "$so_proxy" ]]; then - set_proxy >> $setup_log 2>&1 - fi } network_init_whiptail() { diff --git a/setup/so-setup b/setup/so-setup index 66542c6f3..0e00b1a60 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -308,7 +308,6 @@ if ! [[ -f $install_opt_file ]]; then break fi done - [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 fi if [[ -n "$TURBO" ]]; then @@ -357,14 +356,14 @@ minion_type=$(get_minion_type) set_default_log_size >> $setup_log 2>&1 if [[ $is_helix ]]; then - RULESETUP=${RULESETUP:-ETOPEN} + RULESETUP=${RULESETUP:-ETOPEN} NSMSETUP=${NSMSETUP:-BASIC} HNSENSOR=${HNSENSOR:-inherit} MANAGERUPDATES=${MANAGERUPDATES:-0} fi if [[ $is_helix || ( $is_manager && $is_node ) ]]; then - RULESETUP=${RULESETUP:-ETOPEN} + RULESETUP=${RULESETUP:-ETOPEN} NSMSETUP=${NSMSETUP:-BASIC} fi @@ -384,7 +383,7 @@ fi if [[ $is_import ]]; then PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-auto} MTU=${MTU:-1500} - RULESETUP=${RULESETUP:-ETOPEN} + RULESETUP=${RULESETUP:-ETOPEN} NSMSETUP=${NSMSETUP:-BASIC} HNSENSOR=${HNSENSOR:-inherit} MANAGERUPDATES=${MANAGERUPDATES:-0} @@ -560,6 +559,8 @@ catch() { # Set REDIRECTIT variable, which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 +[[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + # Begin install { From 4339ded17f4ed27c0b39a62e8313a17f5624682a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:10:14 -0500 Subject: [PATCH 21/65] [wip][fix] Don't add logic to so-setup, create wrapper function in so-functions --- setup/so-functions | 12 ++++++++++++ setup/so-setup | 16 ++-------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e082852e6..ff58bfc09 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -536,6 +536,18 @@ collect_patch_schedule_name_import() { } collect_proxy() { + collect_proxy_details + while ! proxy_validate; do + if whiptail_invalid_proxy; then + collect_proxy_details no_ask + else + so_proxy="" + break + fi + done +} + +collect_proxy_details() { local ask=${1:-true} local use_proxy if [[ $ask != true ]]; then diff --git a/setup/so-setup b/setup/so-setup index 0e00b1a60..8b42ca7b3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -200,11 +200,7 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" > "$net_init_file" if [[ $is_manager ]]; then collect_proxy - while ! proxy_validate; do - whiptail_invalid_proxy - collect_proxy no_ask - done - [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi whiptail_net_setup_complete else @@ -300,14 +296,6 @@ if ! [[ -f $install_opt_file ]]; then if [[ $is_manager ]]; then collect_proxy - while ! proxy_validate; do - if whiptail_invalid_proxy; then - collect_proxy no_ask - else - proxy_addr="" - break - fi - done fi if [[ -n "$TURBO" ]]; then @@ -559,7 +547,7 @@ catch() { # Set REDIRECTIT variable, which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 -[[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 +[[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 # Begin install From b26585464478cbdf2870497bd92d34fd30866f16 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:10:42 -0500 Subject: [PATCH 22/65] [wip] Move proxy config to separate file --- setup/so-functions | 6 +++--- setup/so-variables | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ff58bfc09..26593cdaa 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2247,7 +2247,7 @@ set_main_ip() { # Add /usr/sbin to everyone's path set_path() { - echo "complete -cf sudo" >> "$profile_d_config_file" + echo "complete -cf sudo" >> /etc/profile.d/securityonion.sh } set_proxy() { @@ -2262,9 +2262,9 @@ set_proxy() { echo "export https_proxy=\"\$http_addr\"" echo "export ftp_proxy=\"\$http_addr\"" echo "export no_proxy=\"${no_proxy_string}\"" - } >> "$profile_d_config_file" + } > /etc/profile.d/so-proxy.sh - source "$profile_d_config_file" + source /etc/profile.d/so-proxy.sh [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' diff --git a/setup/so-variables b/setup/so-variables index cd6478fa7..a2fdf03c6 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -72,5 +72,3 @@ export install_opt_file net_init_file=/root/net_init export net_init_file - -export profile_d_config_file='/etc/profile.d/securityonion.sh' From 2ee8c7ad1c7fe00d66e48904a26d0ff85b577ce5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:16:23 -0500 Subject: [PATCH 23/65] [fix] Always pass $proxy_addr since we retry the surrounding function --- setup/so-functions | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 26593cdaa..0986c8899 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -558,8 +558,7 @@ collect_proxy_details() { fi if [[ $use_proxy == 0 ]]; then - - whiptail_proxy_addr + whiptail_proxy_addr "$proxy_addr" while ! valid_proxy "$proxy_addr"; do whiptail_invalid_input From a435ea77e830ef10762a7bb5b09aa7e397678f6f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:43:42 -0500 Subject: [PATCH 24/65] [fix] Also add hostname to no_proxy list --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 0986c8899..ea5720a03 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2252,7 +2252,7 @@ set_path() { set_proxy() { # Don't proxy localhost, local ip, and management ip - local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}" + local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}" # Set proxy environment variables used by curl, wget, docker, and others { From b3c7760ad4210d7c63f43493bdfffeac951c8470 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 14:08:21 -0500 Subject: [PATCH 25/65] [fix] Use correct variable in so-proxy.sh --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ea5720a03..0b35c6e5f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2258,8 +2258,8 @@ set_proxy() { { echo "export use_proxy=on" echo "export http_proxy=\"${so_proxy}\"" - echo "export https_proxy=\"\$http_addr\"" - echo "export ftp_proxy=\"\$http_addr\"" + echo "export https_proxy=\"\$http_proxy\"" + echo "export ftp_proxy=\"\$http_proxy\"" echo "export no_proxy=\"${no_proxy_string}\"" } > /etc/profile.d/so-proxy.sh From e0d0baafcc8453011dc21a61eb62477099c11008 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 16:40:32 -0500 Subject: [PATCH 26/65] [fix] Permanently set proxy for yum using template --- salt/yum/etc/yum.conf.jinja | 2 ++ setup/so-functions | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/yum/etc/yum.conf.jinja b/salt/yum/etc/yum.conf.jinja index 5e1d30510..710b3968b 100644 --- a/salt/yum/etc/yum.conf.jinja +++ b/salt/yum/etc/yum.conf.jinja @@ -14,4 +14,6 @@ clean_requirements_on_remove=1 {% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') %} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 +{% elif salt['pillar.get']('manager:proxy') is not None %} +proxy={{salt['pillar.get']('manager:proxy')}} {% endif %} diff --git a/setup/so-functions b/setup/so-functions index 0b35c6e5f..6d1c8cf27 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1481,6 +1481,7 @@ manager_pillar() { "manager:"\ " mainip: '$MAINIP'"\ " mainint: '$MNIC'"\ + " proxy: '$so_proxy'"\ " esheap: '$ES_HEAP_SIZE'"\ " esclustername: '{{ grains.host }}'"\ " freq: 0"\ @@ -1560,7 +1561,6 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$so_proxy'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ From 33b2bd33fe7826a5a9c4966d0ebc83c4219a8bf9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 17:12:10 -0500 Subject: [PATCH 27/65] [fix] Also create config.json so containers use proxy --- setup/so-functions | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 6d1c8cf27..8057500be 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2265,7 +2265,7 @@ set_proxy() { source /etc/profile.d/so-proxy.sh - [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' + [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p /etc/systemd/system/docker.service.d # Create proxy config for dockerd printf '%s\n'\ @@ -2277,6 +2277,22 @@ set_proxy() { systemctl daemon-reload command -v docker &> /dev/null && systemctl restart docker + # Create config.json for docker containers + [[ -d /root/.docker ]] || mkdir /root/.docker + printf '%s\n'\ + "{"\ + " \"proxies\":"\ + " {"\ + " \"default\":"\ + " {"\ + " \"httpProxy\":\"${so_proxy}\","\ + " \"httpsProxy\":\"${so_proxy}\","\ + " \"ftpProxy\":\"${so_proxy}\","\ + " \"noProxy\":\"${no_proxy_string}\""\ + " }"\ + " }"\ + "}" > /root/.docker/config.json + # Set proxy for package manager if [ "$OS" = 'centos' ]; then echo "proxy=$so_proxy" >> /etc/yum.conf From ec04145d1525ea7be484c466849dc3b731299dd3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Mar 2021 11:34:31 -0500 Subject: [PATCH 28/65] [fix] Set proxy for idstools container manually --- salt/idstools/init.sls | 7 +++++++ setup/so-functions | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index 2d3128797..f65f93a7a 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -19,6 +19,7 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set ENGINE = salt['pillar.get']('global:mdengine', '') %} +{% set proxy = salt['pillar.get']('manager:proxy') %} # IDSTools Setup idstoolsdir: file.directory: @@ -71,6 +72,12 @@ so-idstools: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }} - hostname: so-idstools - user: socore + {% if proxy is not none %} + - environment: + - http_proxy={{ proxy }} + - https_proxy={{ proxy }} + - no_proxy={{ salt['pillar.get']('manager:no_proxy') }} + {% endif %} - binds: - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - /opt/so/rules/nids:/opt/so/rules/nids:rw diff --git a/setup/so-functions b/setup/so-functions index 8057500be..54f22cc44 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1482,6 +1482,7 @@ manager_pillar() { " mainip: '$MAINIP'"\ " mainint: '$MNIC'"\ " proxy: '$so_proxy'"\ + " no_proxy: '$no_proxy_string'"\ " esheap: '$ES_HEAP_SIZE'"\ " esclustername: '{{ grains.host }}'"\ " freq: 0"\ @@ -2252,7 +2253,7 @@ set_path() { set_proxy() { # Don't proxy localhost, local ip, and management ip - local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}" + no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}" # Set proxy environment variables used by curl, wget, docker, and others { From 245902326f5d21740a0d3f1682bfdce7495bd028 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Mar 2021 12:53:20 -0500 Subject: [PATCH 29/65] [wip] Add automation support for proxy settings --- setup/automation/standalone-iso-proxy | 78 +++++++++++++++++++++++++++ setup/so-functions | 4 ++ 2 files changed, 82 insertions(+) create mode 100644 setup/automation/standalone-iso-proxy diff --git a/setup/automation/standalone-iso-proxy b/setup/automation/standalone-iso-proxy new file mode 100644 index 000000000..98bb6c2c6 --- /dev/null +++ b/setup/automation/standalone-iso-proxy @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +proxy_addr=http://10.66.166.30:3128 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/so-functions b/setup/so-functions index 54f22cc44..d677fe9cd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1835,6 +1835,10 @@ proxy_validate() { if [[ $ret != 0 ]]; then error "Could not reach $test_url using proxy $so_proxy" + if [[ -n $TESTING ]]; then + error "Exiting setup" + kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 + fi fi return $ret } From 50fcdb65a6135a5e2e4c2992b0fb2224c9954fe2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Mar 2021 13:53:48 -0500 Subject: [PATCH 30/65] [fix] Modify the proxy automated test * It makes more sense to test the proxy using a network install, not via the iso --- .../{standalone-iso-proxy => standalone-net-centos-proxy} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename setup/automation/{standalone-iso-proxy => standalone-net-centos-proxy} (98%) diff --git a/setup/automation/standalone-iso-proxy b/setup/automation/standalone-net-centos-proxy similarity index 98% rename from setup/automation/standalone-iso-proxy rename to setup/automation/standalone-net-centos-proxy index 98bb6c2c6..e8aae3813 100644 --- a/setup/automation/standalone-iso-proxy +++ b/setup/automation/standalone-net-centos-proxy @@ -17,7 +17,7 @@ TESTING=true -address_type=DHCP +# address_type=DHCP ADMINUSER=onionuser ADMINPASS1=onionuser ADMINPASS2=onionuser From e5110dc3fcaa13d02ddf9d9d38e47fa843e695e1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Mar 2021 14:08:03 -0500 Subject: [PATCH 31/65] [fix] None -> none --- salt/yum/etc/yum.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/yum/etc/yum.conf.jinja b/salt/yum/etc/yum.conf.jinja index 710b3968b..bc276d84d 100644 --- a/salt/yum/etc/yum.conf.jinja +++ b/salt/yum/etc/yum.conf.jinja @@ -14,6 +14,6 @@ clean_requirements_on_remove=1 {% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') %} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 -{% elif salt['pillar.get']('manager:proxy') is not None %} +{% elif salt['pillar.get']('manager:proxy') is not none %} proxy={{salt['pillar.get']('manager:proxy')}} {% endif %} From 35ae9363f541fd5b9857e90d45a6c926b220b659 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Mar 2021 20:15:37 -0500 Subject: [PATCH 32/65] [fix] Log gateway error, and don't show whiptail msg on automated installs --- setup/so-functions | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 47fd8e07f..a85ab1982 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -792,12 +792,14 @@ check_sos_appliance() { compare_main_nic_ip() { if ! [[ $MNIC =~ ^(tun|wg|vpn).*$ ]]; then if [[ "$MAINIP" != "$MNIC_IP" ]]; then + error "[ERROR] Main gateway ($MAINIP) does not match ip address of managament NIC ($MNIC_IP)." + read -r -d '' message <<- EOM The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). This is not a supported configuration, please remediate and rerun setup. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 + [[ -n $TESTING ]] || whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi else @@ -1051,11 +1053,11 @@ disable_ipv6() { sysctl -w net.ipv6.conf.all.disable_ipv6=1 sysctl -w net.ipv6.conf.default.disable_ipv6=1 } >> "$setup_log" 2>&1 - { - echo "net.ipv6.conf.all.disable_ipv6 = 1" - echo "net.ipv6.conf.default.disable_ipv6 = 1" - echo "net.ipv6.conf.lo.disable_ipv6 = 1" - } >> /etc/sysctl.conf + { + echo "net.ipv6.conf.all.disable_ipv6 = 1" + echo "net.ipv6.conf.default.disable_ipv6 = 1" + echo "net.ipv6.conf.lo.disable_ipv6 = 1" + } >> /etc/sysctl.conf } #disable_misc_network_features() { From ca2766511b1cb33574f91ad1ddb8cbfecc25bd6e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 09:02:53 -0500 Subject: [PATCH 33/65] Revert "[wip] Change when proxy is set up so main ip is known" This reverts commit 1ea3cb1c615fc876b1429ada48bca2c81f62ecd7. # Conflicts: # setup/so-functions --- setup/so-functions | 6 ++---- setup/so-setup | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a85ab1982..7951430db 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -795,7 +795,7 @@ compare_main_nic_ip() { error "[ERROR] Main gateway ($MAINIP) does not match ip address of managament NIC ($MNIC_IP)." read -r -d '' message <<- EOM - The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). + The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). This is not a supported configuration, please remediate and rerun setup. EOM @@ -1743,8 +1743,6 @@ network_init() { if [[ "$setup_type" == 'iso' ]]; then set_management_interface fi - set_main_ip >> $setup_log 2>&1 - compare_main_nic_ip } network_init_whiptail() { @@ -2295,7 +2293,7 @@ set_proxy() { "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload - command -v docker &> /dev/null && systemctl restart docker + systemctl restart docker # Create config.json for docker containers [[ -d /root/.docker ]] || mkdir /root/.docker diff --git a/setup/so-setup b/setup/so-setup index c7ebddd78..dd98d1730 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -545,7 +545,9 @@ catch() { exit } -# Set REDIRECTIT variable, which is used by a function outside the below subshell +# This block sets REDIRECTIT which is used by a function outside the below subshell +set_main_ip >> $setup_log 2>&1 +compare_main_nic_ip set_redirect >> $setup_log 2>&1 [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 From 368b04b24ed5ab6f7dc2cfc04f2b127539d0f326 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 09:04:17 -0500 Subject: [PATCH 34/65] Add back accidentally removed code --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 7951430db..c3f3503d9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2293,7 +2293,7 @@ set_proxy() { "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload - systemctl restart docker + command -v docker &> /dev/null && systemctl restart docker # Create config.json for docker containers [[ -d /root/.docker ]] || mkdir /root/.docker From 5c7d3656dd5ec2be4d2c3ea57a168166f5c1a62d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 12:26:17 -0500 Subject: [PATCH 35/65] [fix] Don't try to create so_proxy during automated installs, just set it --- setup/automation/standalone-net-centos-proxy | 2 +- setup/so-functions | 3 ++- setup/so-whiptail | 7 +++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/setup/automation/standalone-net-centos-proxy b/setup/automation/standalone-net-centos-proxy index e8aae3813..b364c64fc 100644 --- a/setup/automation/standalone-net-centos-proxy +++ b/setup/automation/standalone-net-centos-proxy @@ -62,7 +62,7 @@ OSQUERY=1 # PATCHSCHEDULEHOURS= PATCHSCHEDULENAME=auto PLAYBOOK=1 -proxy_addr=http://10.66.166.30:3128 +so_proxy=http://10.66.166.30:3128 # REDIRECTHOST= REDIRECTINFO=IP RULESETUP=ETOPEN diff --git a/setup/so-functions b/setup/so-functions index c3f3503d9..300d384d0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -536,6 +536,7 @@ collect_patch_schedule_name_import() { } collect_proxy() { + [[ -n $TESTING ]] && return collect_proxy_details while ! proxy_validate; do if whiptail_invalid_proxy; then @@ -571,7 +572,7 @@ collect_proxy_details() { local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do - if echo "$proxy_addr" | grep "$prefix"; then + if echo "$proxy_addr" | grep -q "$prefix"; then local proxy=${proxy_addr#"$prefix"} so_proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}" break diff --git a/setup/so-whiptail b/setup/so-whiptail index f62d4c32b..3423dfbe5 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1237,6 +1237,7 @@ whiptail_proxy_ask() { whiptail_proxy_addr() { [ -n "$TESTING" ] && return + local message read -r -d '' message <<- EOM Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234). @@ -1260,12 +1261,18 @@ whiptail_proxy_auth_user() { [ -n "$TESTING" ] && return proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus } whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus } whiptail_requirements_error() { From ed4f8025beea16f5000f04081c3d4ffcd280bfc6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 13:57:24 -0500 Subject: [PATCH 36/65] [fix] Also check for proxy to be empty string --- salt/yum/etc/yum.conf.jinja | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/yum/etc/yum.conf.jinja b/salt/yum/etc/yum.conf.jinja index bc276d84d..d381f120f 100644 --- a/salt/yum/etc/yum.conf.jinja +++ b/salt/yum/etc/yum.conf.jinja @@ -1,3 +1,4 @@ +{% set proxy = salt['pillar.get']('manager:proxy') -%} [main] cachedir=/var/cache/yum/$basearch/$releasever keepcache=0 @@ -11,9 +12,8 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release clean_requirements_on_remove=1 - -{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') %} +{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') -%} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 -{% elif salt['pillar.get']('manager:proxy') is not none %} -proxy={{salt['pillar.get']('manager:proxy')}} +{% elif proxy != '' and proxy is not none -%} +proxy={{ proxy }} {% endif %} From 5ee6856a07c3c6842a1393a142aaf4dfff5542ba Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 15:43:54 -0500 Subject: [PATCH 37/65] Strip the last substring following a hyphen for automated branches Also don't show the user a stack trace on invalid version strings, just alert on the bad string and exit --- salt/common/tools/sbin/so-docker-prune | 31 +++++++++++++++++--------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 06e05d7dc..4a7bd4239 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -16,7 +16,7 @@ # along with this program. If not, see . import sys, argparse, re, docker -from packaging.version import Version +from packaging.version import Version, InvalidVersion from itertools import groupby, chain @@ -34,6 +34,11 @@ def get_image_version(string) -> str: # Version doesn't like "latest", so use a high semver return '999999.9.9' else: + try: + Version(ver) + except InvalidVersion: + # Strip the last substring following a hyphen for automated branches + ver = '-'.join(ver.split('-')[:-1]) return ver @@ -54,16 +59,20 @@ def main(quiet): no_prunable = True for t_list in grouped_tag_lists: - # Keep the 2 most current images - t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True) - if len(t_list) <= 2: - continue - else: - no_prunable = False - for tag in t_list[2:]: - if not quiet: print(f'Removing image {tag}') - client.images.remove(tag) - + try: + # Keep the 2 most current images + t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True) + if len(t_list) <= 2: + continue + else: + no_prunable = False + for tag in t_list[2:]: + if not quiet: print(f'Removing image {tag}') + client.images.remove(tag) + except InvalidVersion as e: + print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}') + exit(1) + if no_prunable and not quiet: print('No Security Onion images to prune') From 8c4e66f7bba787d0b464a3641fc56bb89130d507 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 15:52:21 -0500 Subject: [PATCH 38/65] [fix] Print error to stderr --- salt/common/tools/sbin/so-docker-prune | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 4a7bd4239..5a56f506d 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -70,7 +70,7 @@ def main(quiet): if not quiet: print(f'Removing image {tag}') client.images.remove(tag) except InvalidVersion as e: - print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}') + print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr) exit(1) if no_prunable and not quiet: From f85f86ccddcb5f189c37d7b461e4b0b15f452863 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 17:25:23 -0500 Subject: [PATCH 39/65] [fix] Check for empty proxy string everywhere --- salt/idstools/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index f65f93a7a..eaf91bd58 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -72,7 +72,7 @@ so-idstools: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }} - hostname: so-idstools - user: socore - {% if proxy is not none %} + {% if proxy != '' and proxy is not none %} - environment: - http_proxy={{ proxy }} - https_proxy={{ proxy }} From ba71b2fbc81abc60bb47883ea2a0062201563330 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Mar 2021 17:36:34 -0500 Subject: [PATCH 40/65] Change proxy Jinja logic (none and empty string are falsy) --- salt/idstools/init.sls | 2 +- salt/yum/etc/yum.conf.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index eaf91bd58..725520928 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -72,7 +72,7 @@ so-idstools: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }} - hostname: so-idstools - user: socore - {% if proxy != '' and proxy is not none %} + {% if proxy %} - environment: - http_proxy={{ proxy }} - https_proxy={{ proxy }} diff --git a/salt/yum/etc/yum.conf.jinja b/salt/yum/etc/yum.conf.jinja index d381f120f..506036421 100644 --- a/salt/yum/etc/yum.conf.jinja +++ b/salt/yum/etc/yum.conf.jinja @@ -14,6 +14,6 @@ distroverpkg=centos-release clean_requirements_on_remove=1 {% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') -%} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 -{% elif proxy != '' and proxy is not none -%} +{% elif proxy -%} proxy={{ proxy }} {% endif %} From 157badf44859f761b37b8f6f540448fdd974c747 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 10 Mar 2021 16:12:59 -0500 Subject: [PATCH 41/65] [fix] Use correct pillar value for api key Fixes #3388 --- salt/common/tools/sbin/so-cortex-user-add | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-cortex-user-add b/salt/common/tools/sbin/so-cortex-user-add index 7b07ddbde..af6ce4647 100755 --- a/salt/common/tools/sbin/so-cortex-user-add +++ b/salt/common/tools/sbin/so-cortex-user-add @@ -30,7 +30,7 @@ fi USER=$1 -CORTEX_KEY=$(lookup_pillar cortexkey) +CORTEX_KEY=$(lookup_pillar cortexorguserkey) CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api" CORTEX_ORG_NAME=$(lookup_pillar cortexorgname) CORTEX_USER=$USER From 9ffbb9d37eec26ee3801652c3bca6737c7e2c9c2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 10 Mar 2021 16:17:10 -0500 Subject: [PATCH 42/65] [fix] Use update so-cortex-user-enable with correct pillar Fixes #3388 --- salt/common/tools/sbin/so-cortex-user-enable | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-cortex-user-enable b/salt/common/tools/sbin/so-cortex-user-enable index fc68e955e..cfc72bd3e 100755 --- a/salt/common/tools/sbin/so-cortex-user-enable +++ b/salt/common/tools/sbin/so-cortex-user-enable @@ -30,7 +30,7 @@ fi USER=$1 -CORTEX_KEY=$(lookup_pillar cortexkey) +CORTEX_KEY=$(lookup_pillar cortexorguserkey) CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api" CORTEX_USER=$USER From 750de6333d3ecfec19c2d0db401100f7f2da056a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 10 Mar 2021 16:24:21 -0500 Subject: [PATCH 43/65] [fix] Remove last bad usage of cortexkey --- salt/nodered/files/so_flows.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/nodered/files/so_flows.json b/salt/nodered/files/so_flows.json index a8a6e2c69..6a0dea7cf 100644 --- a/salt/nodered/files/so_flows.json +++ b/salt/nodered/files/so_flows.json @@ -1,4 +1,4 @@ {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%} {%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') -%} -{%- set CORTEXKEY = salt['pillar.get']('global:cortexkey', '') -%} +{%- set CORTEXKEY = salt['pillar.get']('global:cortexorgusekey', '') -%} [{"id":"dca608c3.7d8af8","type":"tab","label":"TheHive - Webhook Events","disabled":false,"info":""},{"id":"4db74fa6.2556d","type":"tls-config","z":"","name":"","cert":"","key":"","ca":"","certname":"","keyname":"","caname":"","servername":"","verifyservercert":false},{"id":"aa6cf50d.a02fc8","type":"http in","z":"dca608c3.7d8af8","name":"TheHive Listener","url":"/thehive","method":"post","upload":false,"swaggerDoc":"","x":120,"y":780,"wires":[["2b92aebb.853dc2","2fce29bb.1b1376","82ad0f08.7a53f"]]},{"id":"2b92aebb.853dc2","type":"debug","z":"dca608c3.7d8af8","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","x":470,"y":940,"wires":[]},{"id":"a4ecb84a.805958","type":"switch","z":"dca608c3.7d8af8","name":"Operation","property":"payload.operation","propertyType":"msg","rules":[{"t":"eq","v":"Creation","vt":"str"},{"t":"eq","v":"Update","vt":"str"},{"t":"eq","v":"Delete","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":580,"y":780,"wires":[["f1e954fd.3c21d8"],["65928861.c90a48"],["a259a26c.a21"]],"outputLabels":["Creation","Update","Delete"]},{"id":"f1e954fd.3c21d8","type":"switch","z":"dca608c3.7d8af8","name":"Creation","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":480,"wires":[["e88b4cc2.f6afe"],["8c54e39.a1b4f2"],["64203fe8.e0ad5"],["3511de51.889a02"],["14544a8b.b6b2f5"],["44c595a4.45d45c"],["3eb4bedf.6e20a2"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact","case_task","case_task_log","action","alert","user"],"info":"No webhook data is received for the following events:\n\n- Creation of Dashboard\n- Creation of Case Templates\n"},{"id":"65928861.c90a48","type":"switch","z":"dca608c3.7d8af8","name":"Update","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":860,"wires":[["eebe1748.1cd348"],["d703adc0.12fd1"],["2b738415.408d4c"],["6d97371a.406348"],["4ae621e1.9ae6"],["5786cee2.98109"],["54077728.447648"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact",null,"case_task","case_task_log","alert","user"]},{"id":"a259a26c.a21","type":"switch","z":"dca608c3.7d8af8","name":"Delete","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":890,"y":1200,"wires":[["60c8bcfb.eff1f4"],["df708bab.348308"],["e9a8650c.e20cc8"]],"outputLabels":["case","case_artifact",""],"info":"Deleting a case task doesnt actually trigger a delete event. It triggers an `update` event where the status = cancelled"},{"id":"54077728.447648","type":"switch","z":"dca608c3.7d8af8","name":"User","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Locked","vt":"str"},{"t":"eq","v":"Ok","vt":"str"}],"checkall":"false","repair":false,"outputs":2,"x":1130,"y":980,"wires":[["9429d6c5.5ac788"],["4e3e091c.d35388"]]},{"id":"9429d6c5.5ac788","type":"function","z":"dca608c3.7d8af8","name":"status: Locked","func":"msg.topic = \"[The Hive] A user account was locked\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1380,"y":972,"wires":[[]],"info":"- User account was locked"},{"id":"4e3e091c.d35388","type":"function","z":"dca608c3.7d8af8","name":"status: Ok","func":"msg.topic = \"[The Hive] A user account was changed\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1360,"y":1020,"wires":[[]],"info":"- User account was unlocked\n- User description was changed\n- User role was changed\n- User API key was added\n- User API key was revoked\n"},{"id":"485f3be.1ffcfc4","type":"function","z":"dca608c3.7d8af8","name":"status: Open","func":"// Fires when a Case is updated AND status = open\n// This can include things like TLP/PAP changes\n\nreturn msg;","outputs":1,"noerr":0,"x":1370,"y":660,"wires":[[]]},{"id":"eebe1748.1cd348","type":"switch","z":"dca608c3.7d8af8","name":"case","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Open","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":1130,"y":740,"wires":[["485f3be.1ffcfc4","e4b7b4bf.2fb828"]],"info":"- A case was modified"},{"id":"8c54e39.a1b4f2","type":"switch","z":"dca608c3.7d8af8","name":"case_artifact: Run Analyzer","property":"payload.object.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1600,"y":340,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["eb8cfeb7.a7118","a5dd8a8a.065b88"]],"info":"# References\n\n\n"},{"id":"2fce29bb.1b1376","type":"function","z":"dca608c3.7d8af8","name":"Add headers","func":"msg.thehive_url = 'https://{{ MANAGERIP }}/thehive';\nmsg.cortex_url = 'https://{{ MANAGERIP }}/cortex';\nmsg.cortex_id = 'CORTEX-SERVER-ID';\nreturn msg;","outputs":1,"noerr":0,"x":350,"y":780,"wires":[["a4ecb84a.805958"]]},{"id":"e4b7b4bf.2fb828","type":"function","z":"dca608c3.7d8af8","name":"status: Resolved","func":"// Fires when a case is closed (resolved)\n\nreturn msg;","outputs":1,"noerr":0,"x":1390,"y":720,"wires":[[]]},{"id":"e88b4cc2.f6afe","type":"function","z":"dca608c3.7d8af8","name":"case","func":"// Fires when a case is created\n// or when a responder is generated against a case\n\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":320,"wires":[[]]},{"id":"64203fe8.e0ad5","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is created\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":400,"wires":[[]]},{"id":"3511de51.889a02","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"// Fires when a case task log is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1163,"y":440,"wires":[[]]},{"id":"14544a8b.b6b2f5","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"// Fires when a Responder or Analyzser is Run on an existing observable\n\nreturn msg;","outputs":1,"noerr":0,"x":1173,"y":480,"wires":[[]]},{"id":"2b738415.408d4c","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"\nreturn msg;","outputs":1,"noerr":0,"x":1170,"y":820,"wires":[[]]},{"id":"3eb4bedf.6e20a2","type":"function","z":"dca608c3.7d8af8","name":"user","func":"// Fires when a user is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1133,"y":560,"wires":[[]]},{"id":"d703adc0.12fd1","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"// Fires when an artifact is updated\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":780,"wires":[[]]},{"id":"6d97371a.406348","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is updated\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":860,"wires":[[]]},{"id":"4ae621e1.9ae6","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is updated\n\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":900,"wires":[[]]},{"id":"60c8bcfb.eff1f4","type":"function","z":"dca608c3.7d8af8","name":"case","func":"//Fires when a case is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":1160,"wires":[[]]},{"id":"df708bab.348308","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"//Fires when a case_artifact is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":1200,"wires":[[]]},{"id":"e9a8650c.e20cc8","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":1240,"wires":[[]]},{"id":"5786cee2.98109","type":"function","z":"dca608c3.7d8af8","name":"alert","func":"//Fires when an alert is updated\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":940,"wires":[[]]},{"id":"44c595a4.45d45c","type":"change","z":"dca608c3.7d8af8","d":true,"name":"Convert Alert Msg to Artifacts","rules":[{"t":"move","p":"payload.object.artifacts","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":1200,"y":520,"wires":[["6dcca25e.04bd2c"]]},{"id":"6dcca25e.04bd2c","type":"split","z":"dca608c3.7d8af8","name":"Split Artifacts","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":1430,"y":520,"wires":[["767c84f2.c9ba2c"]]},{"id":"767c84f2.c9ba2c","type":"switch","z":"dca608c3.7d8af8","name":"alert: Run Analyzer","property":"payload.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1630,"y":400,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["a5dd8a8a.065b88","eb8cfeb7.a7118"]],"info":"# References\n\n\n"},{"id":"82ad0f08.7a53f","type":"http response","z":"dca608c3.7d8af8","name":"Ack Event Receipt","statusCode":"200","headers":{},"x":250,"y":940,"wires":[]},{"id":"a5dd8a8a.065b88","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: CERT DNS","func":"msg.analyzer_id = \"4f28afc20d78f98df425e36e561af33f\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1930,"y":420,"wires":[["f050a09f.b2201"]]},{"id":"eb8cfeb7.a7118","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: Urlscan","func":"msg.analyzer_id = \"54e51b62c6c8ddc3cbc3cbdd889a0557\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1920,"y":320,"wires":[["f050a09f.b2201"]]},{"id":"1c448528.3032fb","type":"http request","z":"dca608c3.7d8af8","name":"Submit to Cortex","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ CORTEXKEY }}"},"x":2450,"y":420,"wires":[["ea6614fb.752a78"]]},{"id":"ea6614fb.752a78","type":"debug","z":"dca608c3.7d8af8","name":"Debug","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","x":2670,"y":360,"wires":[]},{"id":"f050a09f.b2201","type":"switch","z":"dca608c3.7d8af8","name":"Cases vs Alerts","property":"tag","propertyType":"msg","rules":[{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"observable","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":2200,"y":360,"wires":[["f7fca977.a73b28"],["1c448528.3032fb"]],"inputLabels":["Data"],"outputLabels":["Cases","Alerts"]},{"id":"f7fca977.a73b28","type":"http request","z":"dca608c3.7d8af8","name":"Submit to TheHive","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ HIVEKEY }}"},"x":2450,"y":280,"wires":[["ea6614fb.752a78"]]}] From 506162bfcc575cd7f1a0bd89ab922c2346bb42a5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 10:52:17 -0500 Subject: [PATCH 44/65] Use auth for automated proxy test --- setup/automation/standalone-net-centos-proxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/automation/standalone-net-centos-proxy b/setup/automation/standalone-net-centos-proxy index b364c64fc..ee2504a98 100644 --- a/setup/automation/standalone-net-centos-proxy +++ b/setup/automation/standalone-net-centos-proxy @@ -62,7 +62,7 @@ OSQUERY=1 # PATCHSCHEDULEHOURS= PATCHSCHEDULENAME=auto PLAYBOOK=1 -so_proxy=http://10.66.166.30:3128 +so_proxy=http://onionuser:0n10nus3r@10.66.166.30:3128 # REDIRECTHOST= REDIRECTINFO=IP RULESETUP=ETOPEN From 0d01f63e3be7dd12e6c2ce81db6cf3e0635764d0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 11:46:46 -0500 Subject: [PATCH 45/65] [fix] Confirm proxy password --- setup/so-functions | 8 +++++++- setup/so-whiptail | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 300d384d0..1f3a6aed0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -568,7 +568,13 @@ collect_proxy_details() { if whiptail_proxy_auth_ask; then whiptail_proxy_auth_user "$proxy_user" - whiptail_proxy_auth_pass # Don't pass in existing pass since it's obfuscated + + local proxy_pass_match=no + while [[ $proxy_pass_match != yes ]]; do + whiptail_proxy_auth_pass # Don't pass in existing pass since it's obfuscated + whiptail_proxy_auth_pass confirm + check_pass_match "$proxy_pass" "$proxy_pass_confirm" "proxy_pass_match" + done local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do diff --git a/setup/so-whiptail b/setup/so-whiptail index 3423dfbe5..051aa60f9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1267,9 +1267,15 @@ whiptail_proxy_auth_user() { } whiptail_proxy_auth_pass() { + local arg=$1 + [ -n "$TESTING" ] && return - proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + if [[ $arg != 'confirm' ]]; then + proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + else + proxy_pass_confirm=$(whiptail --title "Security Onion Setup" --passwordbox "Please confirm the proxy password:" 8 60 3>&1 1>&2 2>&3) + fi local exitstatus=$? whiptail_check_exitstatus $exitstatus From e60bc87ffa1e95b47f8abb3dc6085f4667d022c7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 15:20:39 -0500 Subject: [PATCH 46/65] Install setup required packages later so that also uses the proxy --- setup/so-functions | 27 +++++++++++++-------- setup/so-setup | 58 +++++++++++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 1f3a6aed0..279a4b8d4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -997,36 +997,43 @@ detect_os() { } +installer_progress_loop() { + local i=0 + while true; do + ((i++)) + set_progress_str "$i" 'Checking that all required packages are installed and enabled...' + [[ $i -gt 0 ]] && sleep 5s + done +} + installer_prereq_packages() { if [ "$OS" == centos ]; then # Print message to stdout so the user knows setup is doing something - echo "Installing required packages to run installer..." # Install bind-utils so the host command exists if [[ ! $is_iso ]]; then if ! command -v host > /dev/null 2>&1; then - yum -y install bind-utils >> "$setup_log" 2>&1 + yum -y install bind-utils >> "$setup_log" 2>&1 fi if ! command -v nmcli > /dev/null 2>&1; then { - yum -y install NetworkManager; - systemctl enable NetworkManager; - systemctl start NetworkManager; + yum -y install NetworkManager + systemctl enable NetworkManager + systemctl start NetworkManager } >> "$setup_log" 2<&1 fi if ! command -v bc > /dev/null 2>&1; then - yum -y install bc >> "$setup_log" 2>&1 + yum -y install bc >> "$setup_log" 2>&1 fi if ! yum versionlock > /dev/null 2>&1; then - yum -y install yum-plugin-versionlock >> "$setup_log" 2>&1 + yum -y install yum-plugin-versionlock >> "$setup_log" 2>&1 fi - else + else logCmd "systemctl enable NetworkManager" logCmd "systemctl start NetworkManager" - fi + fi elif [ "$OS" == ubuntu ]; then # Print message to stdout so the user knows setup is doing something - echo "Installing required packages to run installer..." retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 # Install network manager so we can do interface stuff if ! command -v nmcli > /dev/null 2>&1; then diff --git a/setup/so-setup b/setup/so-setup index dd98d1730..8aeda01e1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -94,16 +94,27 @@ if ! [ -f $install_opt_file ]; then fi automated=no -function progress() { - local title='Security Onion Install' +progress() { + local title='Security Onion Setup' + local msg=${1:-'Please wait while installing...'} if [ $automated == no ]; then - whiptail --title "$title" --gauge 'Please wait while installing...' 6 60 0 # append to text + whiptail --title "$title" --gauge "$msg" 6 70 0 # append to text else cat >> $setup_log 2>&1 fi } +# Set up handler for setup to exit early (use `kill -SIGUSR1 "$setup_proc"; exit 1` in child scripts) +trap 'catch $LINENO' SIGUSR1 +setup_proc="$$" +catch() { + info "Fatal error occurred at $1 in so-setup, failing setup." + grep --color=never "ERROR" "$setup_log" > "$error_log" + whiptail_setup_failed + exit 1 +} + detect_os if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then @@ -198,17 +209,14 @@ if ! [[ -f $install_opt_file ]]; then printf '%s\n' \ "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" > "$net_init_file" - if [[ $is_manager ]]; then - collect_proxy - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 - fi + collect_proxy + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 whiptail_net_setup_complete else - whiptail_install_type + true fi - else - whiptail_install_type - fi + fi + whiptail_install_type else source $install_opt_file fi @@ -261,6 +269,24 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then fi fi +if [[ $is_airgap != true ]]; then + collect_proxy +fi +{ + installer_progress_loop & # Run progress bar to 100 in ~8 minutes + progress_bg_proc=$! + installer_prereq_packages + install_success=$? + kill -9 $progress_bg_proc + if [[ $install_success -gt 0 ]]; then + error "Could not install packages required for setup, exiting now." + kill -SIGUSR1 "$setup_proc"; exit 1 + fi + set_progress_str 99 "Detecting whether setup is running in the cloud." + detect_cloud +} | progress '...' + + if ! [[ -f $install_opt_file ]]; then if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone" @@ -535,16 +561,6 @@ whiptail_make_changes # From here on changes will be made. echo "1" > /root/accept_changes -# Set up handler for setup to exit early (use `kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1` in child scripts) -trap 'catch $LINENO' SIGUSR1 - -catch() { - info "Fatal error occurred at $1 in so-setup, failing setup." - grep --color=never "ERROR" "$setup_log" > "$error_log" - whiptail_setup_failed - exit -} - # This block sets REDIRECTIT which is used by a function outside the below subshell set_main_ip >> $setup_log 2>&1 compare_main_nic_ip From 66da3e380f5306db1e2986d90e3ba69b27123c2a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 15:25:38 -0500 Subject: [PATCH 47/65] [fix] Set percentage value when needed --- setup/so-setup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/so-setup b/setup/so-setup index 8aeda01e1..1e96a8b27 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -27,6 +27,8 @@ original_args=("$@") cd "$(dirname "$0")" || exit 255 +echo "Getting started..." + # Source the generic function libraries that are also used by the product after # setup. These functions are intended to be reusable outside of the setup process. source ../salt/common/tools/sbin/so-common @@ -272,6 +274,7 @@ fi if [[ $is_airgap != true ]]; then collect_proxy fi +percentage=0 { installer_progress_loop & # Run progress bar to 100 in ~8 minutes progress_bg_proc=$! From 5612fc10d4bb80b1149d2e307e9cae0e088ab7ec Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 15:53:04 -0500 Subject: [PATCH 48/65] [feat] Remove setup dependency on bc --- setup/so-functions | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 279a4b8d4..0b97098cc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -747,10 +747,10 @@ check_requirements() { else req_storage=100 fi - if (( $(echo "$free_space_root < $req_storage" | bc -l) )); then + if [[ $free_space_root -lt $req_storage ]]; then whiptail_storage_requirements "/" "${free_space_root} GB" "${req_storage} GB" fi - if (( $(echo "$free_space_nsm < $req_storage" | bc -l) )); then + if [[ $free_space_nsm -lt $req_storage ]]; then whiptail_storage_requirements "/nsm" "${free_space_nsm} GB" "${req_storage} GB" fi else @@ -759,7 +759,7 @@ check_requirements() { else req_storage=200 fi - if (( $(echo "$free_space_root < $req_storage" | bc -l) )); then + if [[ $free_space_root -lt $req_storage ]]; then whiptail_storage_requirements "/" "${free_space_root} GB" "${req_storage} GB" fi fi @@ -1022,9 +1022,6 @@ installer_prereq_packages() { systemctl start NetworkManager } >> "$setup_log" 2<&1 fi - if ! command -v bc > /dev/null 2>&1; then - yum -y install bc >> "$setup_log" 2>&1 - fi if ! yum versionlock > /dev/null 2>&1; then yum -y install yum-plugin-versionlock >> "$setup_log" 2>&1 fi @@ -1043,7 +1040,7 @@ installer_prereq_packages() { systemctl start NetworkManager } >> "$setup_log" 2<&1 fi - retry 50 10 "apt-get -y install bc curl" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install curl" >> "$setup_log" 2>&1 || exit 1 fi } From a86b2ab65374cb20c89135de624318cd81013261 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 15:54:46 -0500 Subject: [PATCH 49/65] [fix] Remove additional collect_proxy call --- setup/so-setup | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 1e96a8b27..e098e4f89 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -322,10 +322,6 @@ if ! [[ -f $install_opt_file ]]; then if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init fi - - if [[ $is_manager ]]; then - collect_proxy - fi if [[ -n "$TURBO" ]]; then use_turbo_proxy From 3306ffa792701ed35065b64d35a3c1b53bf9780b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 16:03:43 -0500 Subject: [PATCH 50/65] Only collect proxy once, include manager in no_proxy value on minions --- setup/so-functions | 3 +++ setup/so-setup | 16 +++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 0b97098cc..4d7019ab1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2282,6 +2282,9 @@ set_proxy() { # Don't proxy localhost, local ip, and management ip no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}" + if [[ -n $MSRV ]] && [[ -n $MSRVIP ]];then + no_proxy_string="${no_proxy_string}, ${MSRVIP}, ${MSRV}" + fi # Set proxy environment variables used by curl, wget, docker, and others { diff --git a/setup/so-setup b/setup/so-setup index e098e4f89..eaac276b8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -271,9 +271,15 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then fi fi -if [[ $is_airgap != true ]]; then - collect_proxy +if [[ $is_minion ]]; then + collect_mngr_hostname fi + +if ! [[ -f $install_opt_file ]] && [[ -z $is_airgap ]]; then + collect_proxy_details + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 +fi + percentage=0 { installer_progress_loop & # Run progress bar to 100 in ~8 minutes @@ -311,10 +317,6 @@ if ! [[ -f $install_opt_file ]]; then source "$net_init_file" fi - if [[ $is_minion ]]; then - collect_mngr_hostname - fi - if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then whiptail_management_interface_setup fi @@ -343,6 +345,7 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ "MSRVIP=$MSRVIP" > "$install_opt_file" + [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi @@ -564,7 +567,6 @@ echo "1" > /root/accept_changes set_main_ip >> $setup_log 2>&1 compare_main_nic_ip set_redirect >> $setup_log 2>&1 -[[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 # Begin install From e2fc1b0b39064be80401f93cf1cce3a237a9dc13 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 16:06:49 -0500 Subject: [PATCH 51/65] Redirect output of kill command --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index eaac276b8..796764cca 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -289,7 +289,7 @@ percentage=0 kill -9 $progress_bg_proc if [[ $install_success -gt 0 ]]; then error "Could not install packages required for setup, exiting now." - kill -SIGUSR1 "$setup_proc"; exit 1 + kill -SIGUSR1 "$setup_proc" &> /dev/null; exit 1 fi set_progress_str 99 "Detecting whether setup is running in the cloud." detect_cloud From 8e8bb1489b1961717b0119ee7caa48e2fc8a4e4f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 16:13:52 -0500 Subject: [PATCH 52/65] Redirect output of kill command --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 796764cca..691ba3871 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -286,7 +286,7 @@ percentage=0 progress_bg_proc=$! installer_prereq_packages install_success=$? - kill -9 $progress_bg_proc + kill -9 $progress_bg_proc &> /dev/null if [[ $install_success -gt 0 ]]; then error "Could not install packages required for setup, exiting now." kill -SIGUSR1 "$setup_proc" &> /dev/null; exit 1 From b35e65190eddb4d1ba63b19c3493d8a80a9db6af Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 16:30:14 -0500 Subject: [PATCH 53/65] [fix] Fix dependency install progress bar --- setup/so-functions | 24 ++++++++++++++++++------ setup/so-setup | 31 +++++++++++-------------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4d7019ab1..d3747f3b5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1001,12 +1001,14 @@ installer_progress_loop() { local i=0 while true; do ((i++)) - set_progress_str "$i" 'Checking that all required packages are installed and enabled...' + set_progress_str "$i" 'Checking that all required packages are installed and enabled...' nolog [[ $i -gt 0 ]] && sleep 5s done } installer_prereq_packages() { + installer_progress_loop & # Run progress bar to 100 in ~8 minutes + progress_bg_proc=$! if [ "$OS" == centos ]; then # Print message to stdout so the user knows setup is doing something @@ -1042,6 +1044,13 @@ installer_prereq_packages() { fi retry 50 10 "apt-get -y install curl" >> "$setup_log" 2>&1 || exit 1 fi + + install_success=$? + kill -9 "$progress_bg_proc" >> "$setup_log" 2>&1 + if [[ $install_success -gt 0 ]]; then + echo "Could not install packages required for setup, exiting now." >> "$setup_log" 2>&1 + kill -SIGUSR1 "$setup_proc" >> "$setup_log" 2>&1; exit 1 + fi } disable_auto_start() { @@ -2369,6 +2378,7 @@ set_progress_str() { local percentage_input=$1 progress_bar_text=$2 export progress_bar_text + local nolog=$2 if (( "$percentage_input" >= "$percentage" )); then percentage="$percentage_input" @@ -2378,12 +2388,14 @@ set_progress_str() { echo -e "$percentage_str" - info "Progressing ($percentage%): $progress_bar_text" + if [[ -z $nolog ]]; then + info "Progressing ($percentage%): $progress_bar_text" - printf '%s\n' \ - '----'\ - "$percentage% - ${progress_bar_text^^}"\ - "----" >> "$setup_log" 2>&1 + # printf '%s\n' \ + # '----'\ + # "$percentage% - ${progress_bar_text^^}"\ + # "----" >> "$setup_log" 2>&1 + fi } set_ssh_cmds() { diff --git a/setup/so-setup b/setup/so-setup index 691ba3871..46b005a5e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -95,6 +95,16 @@ if ! [ -f $install_opt_file ]; then analyze_system fi +# Set up handler for setup to exit early (use `kill -SIGUSR1 "$setup_proc"; exit 1` in child scripts) +trap 'catch $LINENO' SIGUSR1 +setup_proc="$$" +catch() { + info "Fatal error occurred at $1 in so-setup, failing setup." + grep --color=never "ERROR" "$setup_log" > "$error_log" + whiptail_setup_failed + exit 1 +} + automated=no progress() { local title='Security Onion Setup' @@ -107,16 +117,6 @@ progress() { fi } -# Set up handler for setup to exit early (use `kill -SIGUSR1 "$setup_proc"; exit 1` in child scripts) -trap 'catch $LINENO' SIGUSR1 -setup_proc="$$" -catch() { - info "Fatal error occurred at $1 in so-setup, failing setup." - grep --color=never "ERROR" "$setup_log" > "$error_log" - whiptail_setup_failed - exit 1 -} - detect_os if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then @@ -167,7 +167,6 @@ set_ssh_cmds $automated local_sbin="$(pwd)/../salt/common/tools/sbin" export PATH=$PATH:$local_sbin -installer_prereq_packages && detect_cloud set_network_dev_status_list if [ "$OS" == ubuntu ]; then @@ -282,16 +281,8 @@ fi percentage=0 { - installer_progress_loop & # Run progress bar to 100 in ~8 minutes - progress_bg_proc=$! installer_prereq_packages - install_success=$? - kill -9 $progress_bg_proc &> /dev/null - if [[ $install_success -gt 0 ]]; then - error "Could not install packages required for setup, exiting now." - kill -SIGUSR1 "$setup_proc" &> /dev/null; exit 1 - fi - set_progress_str 99 "Detecting whether setup is running in the cloud." + set_progress_str 99 "Detecting whether setup is running in the cloud." nolog detect_cloud } | progress '...' From b14b9e8e1743108003db9c33fe6553ebd4f98071 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 16:34:54 -0500 Subject: [PATCH 54/65] [fix] Fix dependency install progress bar --- setup/so-functions | 10 ---------- setup/so-setup | 9 +++++++++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d3747f3b5..d4d1b4f96 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1007,9 +1007,6 @@ installer_progress_loop() { } installer_prereq_packages() { - installer_progress_loop & # Run progress bar to 100 in ~8 minutes - progress_bg_proc=$! - if [ "$OS" == centos ]; then # Print message to stdout so the user knows setup is doing something # Install bind-utils so the host command exists @@ -1044,13 +1041,6 @@ installer_prereq_packages() { fi retry 50 10 "apt-get -y install curl" >> "$setup_log" 2>&1 || exit 1 fi - - install_success=$? - kill -9 "$progress_bg_proc" >> "$setup_log" 2>&1 - if [[ $install_success -gt 0 ]]; then - echo "Could not install packages required for setup, exiting now." >> "$setup_log" 2>&1 - kill -SIGUSR1 "$setup_proc" >> "$setup_log" 2>&1; exit 1 - fi } disable_auto_start() { diff --git a/setup/so-setup b/setup/so-setup index 46b005a5e..0f37de3d4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -281,7 +281,16 @@ fi percentage=0 { + installer_progress_loop & # Run progress bar to 100 in ~8 minutes + progress_bg_proc=$! installer_prereq_packages + install_success=$? + kill -9 "$progress_bg_proc" + wait "$progress_bg_proc" &> /dev/null # Kill just sends signal, redirect output of wait to catch stdout + if [[ $install_success -gt 0 ]]; then + echo "Could not install packages required for setup, exiting now." >> "$setup_log" 2>&1 + kill -SIGUSR1 "$setup_proc"; exit 1 + fi set_progress_str 99 "Detecting whether setup is running in the cloud." nolog detect_cloud } | progress '...' From 6d431c0bdaff1e4a4bf51632da514ed85e52d78e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 16:36:56 -0500 Subject: [PATCH 55/65] Add more info to comment --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 0f37de3d4..3fbb1d681 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -281,7 +281,7 @@ fi percentage=0 { - installer_progress_loop & # Run progress bar to 100 in ~8 minutes + installer_progress_loop & # Run progress bar to 100 in ~8 minutes while waiting for package installs progress_bg_proc=$! installer_prereq_packages install_success=$? From 68422049816b0a361c348a3ce41be4b989358f98 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Mar 2021 16:55:06 -0500 Subject: [PATCH 56/65] Ask for hostname earlier in setup --- setup/so-functions | 2 -- setup/so-setup | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d4d1b4f96..fb5706ec4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1758,7 +1758,6 @@ network_init() { network_init_whiptail() { case "$setup_type" in 'iso') - collect_hostname whiptail_management_nic whiptail_dhcp_or_static @@ -1772,7 +1771,6 @@ network_init_whiptail() { 'network') whiptail_network_notice whiptail_dhcp_warn - collect_hostname whiptail_management_nic ;; esac diff --git a/setup/so-setup b/setup/so-setup index 3fbb1d681..d7cbec540 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -204,6 +204,7 @@ if ! [[ -f $install_opt_file ]]; then if [[ $setup_type == 'iso' ]] && [ "$automated" == no ]; then whiptail_first_menu_iso if [[ $option == "CONFIGURENETWORK" ]]; then + collect_hostname network_init_whiptail whiptail_management_interface_setup network_init @@ -216,7 +217,8 @@ if ! [[ -f $install_opt_file ]]; then else true fi - fi + fi + collect_hostname whiptail_install_type else source $install_opt_file From f80dfda60b1ab4223187b095f1fbbf13787c982a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 12 Mar 2021 11:39:44 -0500 Subject: [PATCH 57/65] Only run initial installer progress to 98 to avoid sitting at 100 --- setup/so-functions | 2 +- setup/so-setup | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fb5706ec4..314d5f77d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1000,7 +1000,7 @@ detect_os() { installer_progress_loop() { local i=0 while true; do - ((i++)) + [[ $i -lt 98 ]] && ((i++)) set_progress_str "$i" 'Checking that all required packages are installed and enabled...' nolog [[ $i -gt 0 ]] && sleep 5s done diff --git a/setup/so-setup b/setup/so-setup index d7cbec540..8eb267379 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -283,7 +283,7 @@ fi percentage=0 { - installer_progress_loop & # Run progress bar to 100 in ~8 minutes while waiting for package installs + installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs progress_bg_proc=$! installer_prereq_packages install_success=$? From 6ae3a26cbe6ceea4857a4e6805332327e08631c7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 15 Mar 2021 12:34:13 -0400 Subject: [PATCH 58/65] Revert all proxy changes on reinstall --- setup/so-functions | 18 ++++++++++++++++++ setup/so-setup | 15 ++++++++++----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 314d5f77d..466365b42 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1946,6 +1946,24 @@ reinstall_init() { } >> "$setup_log" 2>&1 } +reset_proxy() { + [[ -f /etc/profile.d/so-proxy.sh ]] && rm -f /etc/profile.d/so-proxy.sh + + [[ -f /etc/systemd/system/docker.service.d/http-proxy.conf ]] && rm -f /etc/systemd/system/docker.service.d/http-proxy.conf + systemctl daemon-reload + command -v docker &> /dev/null && systemctl restart docker + + [[ -f /root/.docker/config.json ]] && rm -f /root/.docker/config.json + + [[ -f /etc/gitconfig ]] && rm -f /etc/gitconfig + + if [[ $OS == 'centos' ]]; then + sed -i "/proxy=/d" /etc/yum.conf + else + [[ -f /etc/apt/apt.conf.d/00-proxy.conf ]] && rm -f /etc/apt/apt.conf.d/00-proxy.conf + fi +} + backup_dir() { dir=$1 backup_suffix=$2 diff --git a/setup/so-setup b/setup/so-setup index 8eb267379..3f39eaaf6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -205,14 +205,15 @@ if ! [[ -f $install_opt_file ]]; then whiptail_first_menu_iso if [[ $option == "CONFIGURENETWORK" ]]; then collect_hostname + reset_prpxy >> $setup_log 2>&1 + collect_proxy + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 network_init_whiptail whiptail_management_interface_setup network_init printf '%s\n' \ "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" > "$net_init_file" - collect_proxy - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 whiptail_net_setup_complete else true @@ -276,9 +277,13 @@ if [[ $is_minion ]]; then collect_mngr_hostname fi -if ! [[ -f $install_opt_file ]] && [[ -z $is_airgap ]]; then - collect_proxy_details - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 + +if ! [[ -f $install_opt_file ]]; then + reset_prpxy >> $setup_log 2>&1 # Always reset proxy when answer file doesn't exist + if [[ -z $is_airgap ]]; then + collect_proxy_details + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 + fi fi percentage=0 From efc0463201a1582ee43370eed3bd710aaf8bb054 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 15 Mar 2021 14:45:23 -0400 Subject: [PATCH 59/65] Change when proxy + variables are set so strings are built correctly --- setup/so-setup | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 3f39eaaf6..12b59a12d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -205,21 +205,22 @@ if ! [[ -f $install_opt_file ]]; then whiptail_first_menu_iso if [[ $option == "CONFIGURENETWORK" ]]; then collect_hostname - reset_prpxy >> $setup_log 2>&1 - collect_proxy - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 network_init_whiptail whiptail_management_interface_setup network_init printf '%s\n' \ "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" > "$net_init_file" + set_main_ip >> $setup_log 2>&1 + compare_main_nic_ip + reset_prpxy >> $setup_log 2>&1 + collect_proxy + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 whiptail_net_setup_complete else true fi fi - collect_hostname whiptail_install_type else source $install_opt_file @@ -273,19 +274,6 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then fi fi -if [[ $is_minion ]]; then - collect_mngr_hostname -fi - - -if ! [[ -f $install_opt_file ]]; then - reset_prpxy >> $setup_log 2>&1 # Always reset proxy when answer file doesn't exist - if [[ -z $is_airgap ]]; then - collect_proxy_details - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 - fi -fi - percentage=0 { installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs @@ -319,6 +307,7 @@ if ! [[ -f $install_opt_file ]]; then [[ -f $net_init_file ]] && whiptail_net_reinit && reinit_networking=true if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + collect_hostname network_init_whiptail else source "$net_init_file" @@ -331,6 +320,19 @@ if ! [[ -f $install_opt_file ]]; then if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init fi + + set_main_ip >> $setup_log 2>&1 + compare_main_nic_ip + + if [[ $is_minion ]]; then + collect_mngr_hostname + fi + + reset_prpxy >> $setup_log 2>&1 + if [[ -z $is_airgap ]]; then + collect_proxy_details + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 + fi if [[ -n "$TURBO" ]]; then use_turbo_proxy @@ -571,8 +573,6 @@ whiptail_make_changes echo "1" > /root/accept_changes # This block sets REDIRECTIT which is used by a function outside the below subshell -set_main_ip >> $setup_log 2>&1 -compare_main_nic_ip set_redirect >> $setup_log 2>&1 From 086f2b34373161cb835bccdb741f4f5e58c165ff Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 15 Mar 2021 14:59:24 -0400 Subject: [PATCH 60/65] Change when prereq packages are installed to follow new order --- setup/so-functions | 17 ++--------------- setup/so-setup | 37 ++++++++++++++++--------------------- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 466365b42..b13e950e2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1008,26 +1008,13 @@ installer_progress_loop() { installer_prereq_packages() { if [ "$OS" == centos ]; then - # Print message to stdout so the user knows setup is doing something - # Install bind-utils so the host command exists if [[ ! $is_iso ]]; then - if ! command -v host > /dev/null 2>&1; then - yum -y install bind-utils >> "$setup_log" 2>&1 - fi - if ! command -v nmcli > /dev/null 2>&1; then - { - yum -y install NetworkManager - systemctl enable NetworkManager - systemctl start NetworkManager - } >> "$setup_log" 2<&1 - fi if ! yum versionlock > /dev/null 2>&1; then yum -y install yum-plugin-versionlock >> "$setup_log" 2>&1 fi - else - logCmd "systemctl enable NetworkManager" - logCmd "systemctl start NetworkManager" fi + logCmd "systemctl enable NetworkManager" + logCmd "systemctl start NetworkManager" elif [ "$OS" == ubuntu ]; then # Print message to stdout so the user knows setup is doing something retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 diff --git a/setup/so-setup b/setup/so-setup index 12b59a12d..0eae93054 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -274,23 +274,6 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then fi fi -percentage=0 -{ - installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs - progress_bg_proc=$! - installer_prereq_packages - install_success=$? - kill -9 "$progress_bg_proc" - wait "$progress_bg_proc" &> /dev/null # Kill just sends signal, redirect output of wait to catch stdout - if [[ $install_success -gt 0 ]]; then - echo "Could not install packages required for setup, exiting now." >> "$setup_log" 2>&1 - kill -SIGUSR1 "$setup_proc"; exit 1 - fi - set_progress_str 99 "Detecting whether setup is running in the cloud." nolog - detect_cloud -} | progress '...' - - if ! [[ -f $install_opt_file ]]; then if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone" @@ -333,10 +316,6 @@ if ! [[ -f $install_opt_file ]]; then collect_proxy_details [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi - - if [[ -n "$TURBO" ]]; then - use_turbo_proxy - fi if [[ $is_minion ]]; then add_mngr_ip_to_hosts @@ -368,6 +347,22 @@ else rm -rf $install_opt_file >> "$setup_log" 2>&1 fi +percentage=0 +{ + installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs + progress_bg_proc=$! + installer_prereq_packages + install_success=$? + kill -9 "$progress_bg_proc" + wait "$progress_bg_proc" &> /dev/null # Kill just sends signal, redirect output of wait to catch stdout + if [[ $install_success -gt 0 ]]; then + echo "Could not install packages required for setup, exiting now." >> "$setup_log" 2>&1 + kill -SIGUSR1 "$setup_proc"; exit 1 + fi + set_progress_str 99 "Detecting whether setup is running in the cloud." nolog + detect_cloud +} | progress '...' + short_name=$(echo "$HOSTNAME" | awk -F. '{print $1}') MINION_ID=$(echo "${short_name}_${install_type}" | tr '[:upper:]' '[:lower:]') From 59c62393b556792cab7d559839a9da3a4e79bd3c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Mar 2021 10:18:02 -0400 Subject: [PATCH 61/65] Change back to validating proxy, show user error message from curl --- setup/so-functions | 11 +++-------- setup/so-setup | 6 +++--- setup/so-whiptail | 3 ++- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index b13e950e2..9667531d7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -568,13 +568,7 @@ collect_proxy_details() { if whiptail_proxy_auth_ask; then whiptail_proxy_auth_user "$proxy_user" - - local proxy_pass_match=no - while [[ $proxy_pass_match != yes ]]; do - whiptail_proxy_auth_pass # Don't pass in existing pass since it's obfuscated - whiptail_proxy_auth_pass confirm - check_pass_match "$proxy_pass" "$proxy_pass_confirm" "proxy_pass_match" - done + whiptail_proxy_auth_pass "$proxy_pass" local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do @@ -1827,11 +1821,12 @@ print_salt_state_apply() { proxy_validate() { local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" - curl "$test_url" --proxy "$so_proxy" &> /dev/null + proxy_test_err=$(curl -s "$test_url" --proxy "$so_proxy" 2>&1) local ret=$? if [[ $ret != 0 ]]; then error "Could not reach $test_url using proxy $so_proxy" + error "Received error: $proxy_test_err" if [[ -n $TESTING ]]; then error "Exiting setup" kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 diff --git a/setup/so-setup b/setup/so-setup index 0eae93054..62b0efb6c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -213,7 +213,7 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" > "$net_init_file" set_main_ip >> $setup_log 2>&1 compare_main_nic_ip - reset_prpxy >> $setup_log 2>&1 + reset_proxy >> $setup_log 2>&1 collect_proxy [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 whiptail_net_setup_complete @@ -311,9 +311,9 @@ if ! [[ -f $install_opt_file ]]; then collect_mngr_hostname fi - reset_prpxy >> $setup_log 2>&1 + reset_proxy >> $setup_log 2>&1 if [[ -z $is_airgap ]]; then - collect_proxy_details + collect_proxy [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 0dc0c0c4f..3916d4716 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -595,7 +595,8 @@ whiptail_invalid_proxy() { read -r -d '' message <<- EOM Could not reach test url using proxy ${proxy_addr}. - Check log (${setup_log}) for details. + Error was: + $proxy_test_err EOM whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 10 60 From 6a48d7f478895061573fe7b758c00b2f574a813e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Mar 2021 11:34:36 -0400 Subject: [PATCH 62/65] Print curl error to populate variable --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 9667531d7..355e48fbe 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1821,7 +1821,7 @@ print_salt_state_apply() { proxy_validate() { local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" - proxy_test_err=$(curl -s "$test_url" --proxy "$so_proxy" 2>&1) + proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" 2>&1) local ret=$? if [[ $ret != 0 ]]; then From c8bbe078a69b6de05891b03200b8ca937dce9488 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Mar 2021 11:42:15 -0400 Subject: [PATCH 63/65] Use more lines on proxy error message --- setup/so-whiptail | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 3916d4716..9753cc290 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -595,11 +595,10 @@ whiptail_invalid_proxy() { read -r -d '' message <<- EOM Could not reach test url using proxy ${proxy_addr}. - Error was: - $proxy_test_err + Error was: ${proxy_test_err} EOM - whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 10 60 + whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 } whiptail_invalid_string() { From 1e7aaf9ffb5a6d9ef3d34f29ac0fa273dc22de81 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Mar 2021 12:32:37 -0400 Subject: [PATCH 64/65] Collect manager info before showing message about copying ssh key --- setup/files/.curlrc | 0 setup/files/.wgetrc | 0 setup/so-setup | 8 +++++--- 3 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 setup/files/.curlrc delete mode 100644 setup/files/.wgetrc diff --git a/setup/files/.curlrc b/setup/files/.curlrc deleted file mode 100644 index e69de29bb..000000000 diff --git a/setup/files/.wgetrc b/setup/files/.wgetrc deleted file mode 100644 index e69de29bb..000000000 diff --git a/setup/so-setup b/setup/so-setup index 62b0efb6c..c23902f8c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -296,6 +296,10 @@ if ! [[ -f $install_opt_file ]]; then source "$net_init_file" fi + if [[ $is_minion ]]; then + collect_mngr_hostname + fi + if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then whiptail_management_interface_setup fi @@ -307,9 +311,7 @@ if ! [[ -f $install_opt_file ]]; then set_main_ip >> $setup_log 2>&1 compare_main_nic_ip - if [[ $is_minion ]]; then - collect_mngr_hostname - fi + reset_proxy >> $setup_log 2>&1 if [[ -z $is_airgap ]]; then From 129db230625fdb17a8dbea2c702a15b55c8df039 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Mar 2021 12:34:44 -0400 Subject: [PATCH 65/65] Move interface message to later in setup --- setup/so-setup | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index c23902f8c..fb4f76be0 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -296,13 +296,6 @@ if ! [[ -f $install_opt_file ]]; then source "$net_init_file" fi - if [[ $is_minion ]]; then - collect_mngr_hostname - fi - - if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then - whiptail_management_interface_setup - fi if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init @@ -311,7 +304,9 @@ if ! [[ -f $install_opt_file ]]; then set_main_ip >> $setup_log 2>&1 compare_main_nic_ip - + if [[ $is_minion ]]; then + collect_mngr_hostname + fi reset_proxy >> $setup_log 2>&1 if [[ -z $is_airgap ]]; then @@ -319,6 +314,10 @@ if ! [[ -f $install_opt_file ]]; then [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi + if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then + whiptail_management_interface_setup + fi + if [[ $is_minion ]]; then add_mngr_ip_to_hosts fi