diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 6861affd7..14d637d50 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -6,8 +6,13 @@ global: managerip: description: The IP address of the grid manager. global: True + advanced: True + regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ + regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: description: What engine to use for meta data generation. Options are ZEEK and SURICATA. + regex: ^(ZEEK|SURICATA)$ + regexFailureMessage: You must enter either ZEEK or SURICATA. global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 8be41b999..4651b7268 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -4,13 +4,15 @@ suricata: threading: set-cpu-affinity: "no" cpu-affinity: - - management-cpu-set: - cpu: [] - - worker-cpu-set: - cpu: [] - mode: exclusive - prio: - default: high + management-cpu-set: + cpu: + - 1 + worker-cpu-set: + cpu: + - 2-3 + mode: exclusive + prio: + default: high af-packet: interface: bond0 cluster-id: 59 @@ -22,32 +24,61 @@ suricata: ring-size: 5000 vars: address-groups: - HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" - EXTERNAL_NET: "any" - HTTP_SERVERS: "$HOME_NET" - SMTP_SERVERS: "$HOME_NET" - SQL_SERVERS: "$HOME_NET" - DNS_SERVERS: "$HOME_NET" - TELNET_SERVERS: "$HOME_NET" - AIM_SERVERS: "$EXTERNAL_NET" - DC_SERVERS: "$HOME_NET" - DNP3_SERVER: "$HOME_NET" - DNP3_CLIENT: "$HOME_NET" - MODBUS_CLIENT: "$HOME_NET" - MODBUS_SERVER: "$HOME_NET" - ENIP_CLIENT: "$HOME_NET" - ENIP_SERVER: "$HOME_NET" + HOME_NET: + - 192.168.0.0/16 + - 10.0.0.0/8 + - 172.16.0.0/12 + EXTERNAL_NET: + - any + HTTP_SERVERS: + - $HOME_NET + SMTP_SERVERS: + - $HOME_NET + SQL_SERVERS: + - $HOME_NET + DNS_SERVERS: + - $HOME_NET + TELNET_SERVERS: + - $HOME_NET + AIM_SERVERS: + - $EXTERNAL_NET + DC_SERVERS: + - $HOME_NET + DNP3_SERVER: + - $HOME_NET + DNP3_CLIENT: + - $HOME_NET + MODBUS_CLIENT: + - $HOME_NET + MODBUS_SERVER: + - $HOME_NET + ENIP_CLIENT: + - $HOME_NET + ENIP_SERVER: + - $HOME_NET port-groups: - HTTP_PORTS: "80" - SHELLCODE_PORTS: "!80" - ORACLE_PORTS: "1521" - SSH_PORTS: "22" - DNP3_PORTS: "20000" - MODBUS_PORTS: "502" - FILE_DATA_PORTS: "[$HTTP_PORTS,110,143]" - FTP_PORTS: "21" - VXLAN_PORTS: "4789" - TEREDO_PORTS: "3544" + HTTP_PORTS: + - 80 + SHELLCODE_PORTS: + - "!80" + ORACLE_PORTS: + - 1521 + SSH_PORTS: + - 22 + DNP3_PORTS: + - 20000 + MODBUS_PORTS: + - 502 + FILE_DATA_PORTS: + - $HTTP_PORTS + - 110 + - 143 + FTP_PORTS: + - 21 + VXLAN_PORTS: + - 4789 + TEREDO_PORTS: + - 3544 default-log-dir: /var/log/suricata/ stats: enabled: "yes" @@ -66,23 +97,23 @@ suricata: community-id: true community-id-seed: 0 types: - - alert: - payload: "no" - payload-buffer-size: 4kb - payload-printable: "yes" - packet: "yes" - metadata: - app-layer: false - flow: false - rule: - metadata: true - raw: true - tagged-packets: "no" - xff: - enabled: "no" - mode: extra-data - deployment: reverse - header: X-Forwarded-For + alert: + payload: "no" + payload-buffer-size: 4kb + payload-printable: "yes" + packet: "yes" + metadata: + app-layer: false + flow: false + rule: + metadata: true + raw: true + tagged-packets: "no" + xff: + enabled: "no" + mode: extra-data + deployment: reverse + header: X-Forwarded-For unified2-alert: enabled: "no" http-log: diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index b238405c8..5576117cc 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -1,4 +1,4 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %} {% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %} {% import_yaml 'suricata/suricata_mdengine.yaml' as suricata_mdengine %} @@ -23,6 +23,45 @@ {% do SURICATAMERGED.config.pop('af-packet') %} {% do SURICATAMERGED.config.update({'af-packet': afpacket}) %} +{# eve-log.types is a list but we convert to dict in defaults to work with ui #} +{# below they are converted back to lists #} +{% load_yaml as evelogtypes %} +{% for le, ld in SURICATAMERGED.config.outputs['eve-log'].types.items() %} + - {{ le }}: {{ ld }} +{% endfor %} +{% endload %} +{% do SURICATAMERGED.config.outputs['eve-log'].pop('types') %} +{% do SURICATAMERGED.config.outputs['eve-log'].update({'types': evelogtypes}) %} + +{# threading.cpu-affinity is a list but we convert to dict in defaults to work with ui #} +{# below they are converted back to lists #} +{% load_yaml as cpuaffinity %} +{% for le, ld in SURICATAMERGED.config.threading['cpu-affinity'].items() %} + - {{ le }}: {{ ld }} +{% endfor %} +{% endload %} +{% do SURICATAMERGED.config.threading.pop('cpu-affinity') %} +{% do SURICATAMERGED.config.threading.update({'cpu-affinity': cpuaffinity}) %} + +{# Find the index of eve-log and file-store in suricata_mdengine.suricata.config.outputs #} +{# update outputs eve-log.types and filestore with config for Suricata metadata engine #} +{% if GLOBALS.md_engine == 'SURICATA' %} +{% for li in suricata_mdengine.suricata.config.outputs %} +{% if 'eve-log' in li.keys() %} +{% do surimeta_evelog_index.append(loop.index0) %} +{% endif %} +{% if 'file-store' in li.keys() %} +{% do surimeta_filestore_index.append(loop.index0) %} +{% endif %} +{% endfor %} +{% set surimeta_evelog_index = surimeta_evelog_index[0] %} +{% set surimeta_filestore_index = surimeta_filestore_index[0] %} +{% do SURICATAMERGED.config.outputs['eve-log'].types.extend(suricata_mdengine.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %} +{% do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %} +{% endif %} + +{# outputs is a list but we convert to dict in defaults to work with ui #} +{# below they are converted back to lists #} {% load_yaml as outputs %} {% for le, ld in SURICATAMERGED.config.outputs.items() %} - {{ le }}: {{ ld }} @@ -31,31 +70,22 @@ {% do SURICATAMERGED.config.pop('outputs') %} {% do SURICATAMERGED.config.update({'outputs': outputs}) %} -{# Find the index of eve-log so it can be updated later #} -{% for li in SURICATAMERGED.config.outputs %} - {% if 'eve-log' in li.keys() %} - {% do default_evelog_index.append(loop.index0) %} - {% endif %} - {% if 'file-store' in li.keys() %} - {% do default_filestore_index.append(loop.index0) %} - {% endif %} +{# change address-groups vars from list to comma seperated string #} +{% for k, v in SURICATAMERGED.config.vars['address-groups'].items() %} +{# if address-group value is a list #} +{% if v is iterable and (v is not string and v is not mapping and v | length > 1) %} +{% do SURICATAMERGED.config.vars['address-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %} +{% else %} +{% do SURICATAMERGED.config.vars['address-groups'].update({k: v[0]}) %} +{% endif %} {% endfor %} -{% set default_evelog_index = default_evelog_index[0] %} -{% set default_filestore_index = default_filestore_index[0] %} -{# Find the index of eve-log so it can be grabbed later #} -{% for li in suricata_mdengine.suricata.config.outputs %} - {% if 'eve-log' in li.keys() %} - {% do surimeta_evelog_index.append(loop.index0) %} - {% endif %} - {% if 'file-store' in li.keys() %} - {% do surimeta_filestore_index.append(loop.index0) %} - {% endif %} +{# change port-groups vars from list to comma seperated string #} +{% for k, v in SURICATAMERGED.config.vars['port-groups'].items() %} +{# if address-group value is a list #} +{% if v is iterable and (v is not string and v is not mapping and v | length > 1) %} +{% do SURICATAMERGED.config.vars['port-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %} +{% else %} +{% do SURICATAMERGED.config.vars['port-groups'].update({k: v[0]}) %} +{% endif %} {% endfor %} -{% set surimeta_evelog_index = surimeta_evelog_index[0] %} -{% set surimeta_filestore_index = surimeta_filestore_index[0] %} - -{% if GLOBALS.md_engine == 'SURICATA' %} - {% do SURICATAMERGED.config.outputs[default_evelog_index]['eve-log'].types.extend(suricata_mdengine.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %} - {% do SURICATAMERGED.config.outputs[default_filestore_index]['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %} -{% endif %} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index f1971f17f..f13e89618 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -12,10 +12,54 @@ suricata: title: SIDS helpLink: suricata.html config: + af-packet: + interface: + description: The network interface that Suricata will monitor. + helpLink: suricata.html + cluster-id: + advanced: True + cluster-type: + advanced: True + regex: ^(cluster_flow|cluster_qm)$ + defrag: + advanced: True + regex: ^(yes|no)$ + use-mmap: + advanced: True + readonly: True + threads: + description: The amount of worker threads. + helpLink: suricata.html + forcedType: int + tpacket-v3: + advanced: True + readonly: True + ring-size: + description: Buffer size for packets per thread. + forcedType: int + helpLink: suricata.html + threading: + set-cpu-affinity: + description: Bind(yes) or unbind(no) management and worker threads to a core or range of cores. + regex: ^(yes|no)$ + helpLink: suricata.html + cpu-affinity: + management-cpu-set: + cpu: + description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to 'yes' for this to be used. + forcedType: "[]string" + helpLink: suricata.html + worker-cpu-set: + cpu: + description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to 'yes' for this to be used. + forcedType: "[]string" + helpLink: suricata.html vars: address-groups: HOME_NET: description: List of hosts or networks. + regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ + regexFailureMessage: You must enter a valid IP address or CIDR. helpLink: suricata.html EXTERNAL_NET: description: List of hosts or networks. @@ -92,19 +136,21 @@ suricata: helpLink: suricata.html outputs: eve-log: - xff: - enabled: - description: Enable X-Forward-For support. - helpLink: suricata.html - mode: - description: Operation mode. This should always be extra-data if you use PCAP. - helpLink: suricata.html - deployment: - description: forward would use the first IP address and reverse would use the last. - helpLink: suricata.html - header: - description: Header name where the actual IP address will be reported. - helpLink: suricata.html + types: + alert: + xff: + enabled: + description: Enable X-Forward-For support. + helpLink: suricata.html + mode: + description: Operation mode. This should always be extra-data if you use PCAP. + helpLink: suricata.html + deployment: + description: forward would use the first IP address and reverse would use the last. + helpLink: suricata.html + header: + description: Header name where the actual IP address will be reported. + helpLink: suricata.html asn1-max-frames: description: Maximum nuber of asn1 frames to decode. helpLink: suricata.html diff --git a/setup/so-functions b/setup/so-functions index 672bc668d..3e2c576ae 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -28,6 +28,12 @@ title() { echo -e "\n-----------------------------\n $1\n-----------------------------\n" >> "$setup_log" 2>&1 } +fail_setup() { + error "Setup encounted an unrecoverable failure, exiting" + touch /root/failure + exit 1 +} + logCmd() { cmd=$1 info "Executing command: $cmd" @@ -796,7 +802,7 @@ compare_main_nic_ip() { EOM [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75 - kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 + kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup fi else # Setup uses MAINIP, but since we ignore the equality condition when using a VPN @@ -921,9 +927,10 @@ create_repo() { detect_cloud() { info "Testing if setup is running on a cloud instance..." - if dmidecode -s bios-version | grep -q amazon || \ - dmidecode -s bios-vendor | grep -q Amazon || \ - dmidecode -s bios-vendor | grep -q Google || \ + if [ -f /etc/SOCLOUD ] || \ + dmidecode -s bios-version 2>&1 | grep -q amazon || \ + dmidecode -s bios-vendor 2>&1 | grep -q Amazon || \ + dmidecode -s bios-vendor 2>&1 | grep -q Google || \ [ -f /var/log/waagent.log ]; then info "Detected a cloud installation..." @@ -943,7 +950,7 @@ detect_os() { pkgman="dnf" else info "We do not support the operating system you are trying to use." - exit 1 + fail_setup fi elif [ -f /etc/os-release ]; then @@ -953,12 +960,12 @@ detect_os() { is_ubuntu=true else info "We do not support your current version of Ubuntu." - exit 1 + fail_setup fi else info "We were unable to determine if you are using a supported OS." - exit 1 + fail_setup fi info "Found OS: $OS $OSVER" @@ -981,7 +988,7 @@ download_elastic_agent_artifacts() { info "Elastic Agent source hash is good." else info "Unable to download the Elastic Agent source files." - exit 1 + fail_setup fi logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" @@ -1012,18 +1019,18 @@ installer_prereq_packages() { if [ "$OS" == ubuntu ]; then # Print message to stdout so the user knows setup is doing something info "Running apt-get update" - retry 150 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1 + retry 150 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || fail_setup # Install network manager so we can do interface stuff if ! command -v nmcli > /dev/null 2>&1; then info "Installing network-manager" - retry 150 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || exit 1 + retry 150 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || fail_setup { systemctl enable NetworkManager systemctl start NetworkManager } >> "$setup_log" 2<&1 fi if ! command -v curl > /dev/null 2>&1; then - retry 150 10 "apt-get -y install curl" >> "$setup_log" 2>&1 || exit 1 + retry 150 10 "apt-get -y install curl" >> "$setup_log" 2>&1 || fail_setup fi fi } @@ -1746,7 +1753,7 @@ proxy_validate() { error "Received error: $proxy_test_err" if [[ -n $TESTING ]]; then error "Exiting setup" - kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 + kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup fi fi return $ret @@ -1817,7 +1824,7 @@ reinstall_init() { # Stop the systemctl process trying to kill the service, show user a message, then exit setup kill -9 $pid - exit 1 + fail_setup fi sleep 5 @@ -2020,7 +2027,7 @@ saltify() { SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //') if [[ $is_ubuntu ]]; then - DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1 + DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || fail_setup update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 local pkg_arr=( 'apache2-utils' @@ -2032,7 +2039,7 @@ saltify() { 'netcat' 'jq' ) - retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1 + retry 150 20 "apt-get -y install ${pkg_arr[*]}" || fail_setup logCmd "mkdir -vp /etc/apt/keyrings" #logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub" @@ -2053,9 +2060,9 @@ saltify() { # Ain't nothing but a GPG - retry 150 20 "apt-get update" "" "Err:" || exit 1 - retry 150 20 "apt-get -y install salt-common-$SALTVERSION salt-minion-$SALTVERSION" || exit 1 - retry 150 20 "apt-mark hold salt-minion salt-common" || exit 1 + retry 150 20 "apt-get update" "" "Err:" || fail_setup + retry 150 20 "apt-get -y install salt-common-$SALTVERSION salt-minion-$SALTVERSION" || fail_setup + retry 150 20 "apt-mark hold salt-minion salt-common" || fail_setup #retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" || exit 1 fi @@ -2122,7 +2129,7 @@ set_main_ip() { info "MAINIP=$MAINIP" info "MNIC_IP=$MNIC_IP" whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit." - exit 1 + fail_setup fi sleep 1 done @@ -2372,13 +2379,13 @@ ubuntu_check() { if [[ $OS == "ubuntu" ]]; then if [[ $waitforstate ]]; then whiptail_ubuntu_notsupported - exit 1 + fail_setup else if [[ $UBUNTUINSTALL == "needtoupgrade" ]]; then whiptail_ubuntu_warning else whiptail_ubuntu_notsupported - exit 1 + fail_setup fi fi fi @@ -2397,9 +2404,9 @@ update_packages() { logCmd "dnf -y update --allowerasing --exclude=salt*,wazuh*,docker*,containerd*" else info "Running apt-get update" - retry 150 10 "apt-get -y update" "" "Err:" >> "$setup_log" 2>&1 || exit 1 + retry 150 10 "apt-get -y update" "" "Err:" >> "$setup_log" 2>&1 || fail_setup info "Running apt-get upgrade" - retry 150 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 + retry 150 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || fail_setup fi } @@ -2445,7 +2452,7 @@ wait_for_file() { } wait_for_salt_minion() { - retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || exit 1 + retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || fail_setup } verify_setup() { diff --git a/setup/so-setup b/setup/so-setup index 4b7ff4d67..d8f07b36a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -10,13 +10,13 @@ uid="$(id -u)" if [ "$uid" -ne 0 ]; then echo "This script must be run using sudo!" - exit 1 + fail_setup fi # Save the original argument array since we modify it original_args=("$@") -cd "$(dirname "$0")" || exit 255 +cd "$(dirname "$0")" || fail_setup echo "Getting started..." @@ -82,7 +82,7 @@ if [[ "$setup_type" == 'iso' ]]; then is_iso=true else echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead." - exit 1 + fail_setup fi fi @@ -161,7 +161,7 @@ catch() { info "Fatal error occurred at $1 in so-setup, failing setup." grep --color=never "ERROR" "$setup_log" > "$error_log" whiptail_setup_failed - exit 1 + fail_setup } # Add the progress function for manager node type installs @@ -236,7 +236,7 @@ case "$setup_type" in ;; *) error "Invalid install type, must be 'iso', 'network' or 'analyst'." - exit 1 + fail_setup ;; esac