From 50fa0dc81ae9e948d67ad09979fadd0b0372806b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Mar 2021 11:32:37 -0400 Subject: [PATCH 001/258] Allow user to enter a description during setup Resolves #2404 --- setup/so-setup | 10 ++++++++-- setup/so-whiptail | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..f20828b85 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -291,8 +291,13 @@ if ! [[ -f $install_opt_file ]]; then [[ -f $net_init_file ]] && whiptail_net_reinit && reinit_networking=true - if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then collect_hostname + fi + + whiptail_node_description + + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init_whiptail else source "$net_init_file" @@ -334,7 +339,8 @@ if ! [[ -f $install_opt_file ]]; then "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ - "MSRVIP=$MSRVIP" > "$install_opt_file" + "MSRVIP=$MSRVIP" \ + "NODE_DESCRIPTION=$NODE_DESCRIPTION" > "$install_opt_file" [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" diff --git a/setup/so-whiptail b/setup/so-whiptail index a0425b5af..bc002085c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1044,6 +1044,16 @@ whiptail_node_advanced() { } +whiptail_node_description() { + [ -n "$TESTING" ] && return + + NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup" \ + --inputbox "Enter a short description for the node or press ENTER to leave blank:" 10 75 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_node_es_heap() { [ -n "$TESTING" ] && return From b83ae4bded9596ededd9dfc087051a9011556dc6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Mar 2021 14:49:14 -0400 Subject: [PATCH 002/258] Upgrade to version 2.3.50 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0f1c3e555..58e1f4fa9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.40 +2.3.50 From 86cb59d5ae0ccb494fbe2fa371da0512b475d3ec Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Mar 2021 14:53:36 -0400 Subject: [PATCH 003/258] Revert upgrade to version 2.3.50 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58e1f4fa9..0f1c3e555 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.50 +2.3.40 From 84485888097243560758d6627222a06b35c64af7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Mar 2021 15:04:02 -0400 Subject: [PATCH 004/258] Upgrade to version 2.3.50 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0f1c3e555..58e1f4fa9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.40 +2.3.50 From 449e0d853ce36c2fb31fd3e35a8bc5cee0306f01 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Mar 2021 15:52:51 -0400 Subject: [PATCH 005/258] Initial support for ntp service via chronyd --- setup/so-functions | 57 ++++++++++++++++++++++++++++++++++++++++++++-- setup/so-setup | 5 ++++ setup/so-variables | 3 +++ setup/so-whiptail | 18 +++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 29a58e718..d5e8c0a6e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -486,6 +486,17 @@ collect_node_ls_pipeline_worker_count() { done } +collect_ntp_servers() { + if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' ]]; then + if whiptail_ntp_ask; then + [[ $is_airgap ]] && ntp_servers="" + whiptail_ntp_servers "$ntp_servers" + else + ntp_servers="" + fi + fi +} + collect_oinkcode() { whiptail_oinkcode @@ -702,6 +713,38 @@ configure_minion() { } >> "$setup_log" 2>&1 } +configure_ntp() { + local chrony_conf=/etc/chrony.conf + + # Install chrony if it isn't already installed + if command -v chronyc &> /dev/null; then + if [ "$OS" == centos ]; then + yum -y install chrony + else + retry 50 10 "apt-get -y install chrony" || exit 1 + fi + fi + + [[ -f $chrony_conf ]] && rm -f $chrony_conf + + # Build list of servers + for addr in "${ntp_servers[@]}"; do + echo "server $addr iburst" >> $chrony_conf + done + + printf '%s\n' \ + 'driftfile /var/lib/chrony/drift' \ + 'makestep 1.0 3' \ + 'rtcsync' \ + 'logdir /var/log/chrony' >> $chrony_conf + + systemctl enable chronyd + systemctl start chronyd + + # Sync time + chronyc -a makestep +} + checkin_at_boot() { local minion_config=/etc/salt/minion @@ -709,6 +752,12 @@ checkin_at_boot() { echo "startup_states: highstate" >> "$minion_config" } +check_ntp_configured() { + if systemctl is-active --quiet chronyd || systemctl is-active --quiet ntpd; then + ntp_configured=true + fi +} + check_requirements() { local standalone_or_dist=$1 local node_type=$2 # optional @@ -1564,12 +1613,16 @@ manager_global() { "global:"\ " soversion: '$SOVERSION'"\ " hnmanager: '$HNMANAGER'"\ - " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ - " managerip: '$MAINIP'" > "$global_pillar" + " managerip: '$MAINIP'" + " ntp_servers:" > "$global_pillar" + + for addr in "${ntp_servers[@]}"; do + echo " - '$addr'" >> "$global_pillar" + done if [[ $is_airgap ]]; then printf '%s\n'\ diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..2082653c5 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -534,6 +534,9 @@ if [[ $is_sensor && ! $is_eval ]]; then fi fi +check_ntp_configured +[[ -z $ntp_configured ]] || collect_ntp_servers + if [[ $is_node && ! $is_eval ]]; then whiptail_node_advanced if [ "$NODESETUP" == 'NODEADVANCED' ]; then @@ -581,6 +584,8 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' + [[ -z $ntp_configured ]] || [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 + reserve_ports set_path diff --git a/setup/so-variables b/setup/so-variables index a2fdf03c6..0a07fc79d 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -72,3 +72,6 @@ export install_opt_file net_init_file=/root/net_init export net_init_file + +ntp_servers="0.pool.ntp.org,1.pool.ntp.org" +export ntp_servers diff --git a/setup/so-whiptail b/setup/so-whiptail index a0425b5af..1ccdf6a90 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1105,6 +1105,24 @@ whiptail_node_ls_pipeline_worker() { } +whiptail_ntp_ask() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "Would you like to configure ntp servers?" 7 44 +} + +whiptail_ntp_servers() { + [ -n "$TESTING" ] && return + + ntp_string=$(whiptail --title "Security Onion Setup" \ + --inputbox "Input the NTP server(s) you would like to use, separated by commas:" 8 75 "$1" 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array +} + whiptail_oinkcode() { [ -n "$TESTING" ] && return From b3f558a1f8481a9144c2d96a50cb3ad8b0c147c9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:14:34 -0400 Subject: [PATCH 006/258] [fix] Also check if proxy is set before asking for ntp servers --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d5e8c0a6e..fd998da14 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -487,7 +487,7 @@ collect_node_ls_pipeline_worker_count() { } collect_ntp_servers() { - if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' ]]; then + if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then if whiptail_ntp_ask; then [[ $is_airgap ]] && ntp_servers="" whiptail_ntp_servers "$ntp_servers" From ace30c07ea5bb75489add30067bb802244764eca Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:22:09 -0400 Subject: [PATCH 007/258] [fix] Also sync time before updating system clock --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index fd998da14..a7a596abe 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -741,7 +741,8 @@ configure_ntp() { systemctl enable chronyd systemctl start chronyd - # Sync time + # Sync time & update the system time + chronyc -a 'burst 4/4' chronyc -a makestep } From 184c763b02d36e78024417cc31edfbe1b181d05f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:36:08 -0400 Subject: [PATCH 008/258] [fix] Export correct variable to check later in setup --- setup/so-functions | 2 +- setup/so-whiptail | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a7a596abe..a346128e1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -592,7 +592,7 @@ collect_proxy_details() { else so_proxy="$proxy_addr" fi - export proxy + export so_proxy fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index 1ccdf6a90..2743ab65b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1289,11 +1289,7 @@ whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - if [[ $arg != 'confirm' ]]; then - proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) - else - proxy_pass_confirm=$(whiptail --title "Security Onion Setup" --passwordbox "Please confirm the proxy password:" 8 60 3>&1 1>&2 2>&3) - fi + proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 0e9c81c145b1229bf82fc4976b55630b4a77e0aa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:44:44 -0400 Subject: [PATCH 009/258] Fix logic around ntp prompt --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 2082653c5..6ed3fa344 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -535,7 +535,7 @@ if [[ $is_sensor && ! $is_eval ]]; then fi check_ntp_configured -[[ -z $ntp_configured ]] || collect_ntp_servers +[[ -z $ntp_configured ]] && collect_ntp_servers if [[ $is_node && ! $is_eval ]]; then whiptail_node_advanced From 2d873b92fa3c19b43b5850220d65270536901a13 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 10:22:41 -0400 Subject: [PATCH 010/258] Fix ntp logic elsewhere --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 6ed3fa344..07eb49500 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -584,7 +584,7 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' - [[ -z $ntp_configured ]] || [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 + [[ -z $ntp_configured ]] && [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 reserve_ports From 9f0afd90f1852d28ad42d126f3cee187d3ef2115 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 11:27:37 -0400 Subject: [PATCH 011/258] [fix] Add missing backslash --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index a346128e1..dbc92aabc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1618,7 +1618,7 @@ manager_global() { " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ - " managerip: '$MAINIP'" + " managerip: '$MAINIP'"\ " ntp_servers:" > "$global_pillar" for addr in "${ntp_servers[@]}"; do From 3287a777a2203a775265c25ca85d9b34ab1acaf7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 11:41:12 -0400 Subject: [PATCH 012/258] [fix] Pre-fill hostname re-enter on default --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index dbc92aabc..ffaa079c1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -400,7 +400,7 @@ collect_hostname() { if [[ $HOSTNAME == 'securityonion' ]]; then # Will only check HOSTNAME=securityonion once if ! (whiptail_avoid_default_hostname); then - whiptail_set_hostname + whiptail_set_hostname "$HOSTNAME" fi fi From 23cd006724755dac1a8fb22e6dd0126d19d3a00c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 14:06:10 -0400 Subject: [PATCH 013/258] so-ssh-harden fixes * Change when script is run during setup * Add newlines to sshd config for legibility --- salt/common/tools/sbin/so-ssh-harden | 8 +++++++- setup/so-functions | 2 -- setup/so-setup | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 1fd7d58d9..5891e4309 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -38,8 +38,12 @@ print_msg() { fi } +# Add newline to ssh for legibility +echo "" >> /etc/ssh/sshd_config + if ! [[ $quiet ]]; then print_sshd_t "ciphers" "Before"; fi sshd -T | grep "^ciphers" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "ciphers" "After" echo "" @@ -51,6 +55,7 @@ fi if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "Before"; fi sshd -T | grep "^kexalgorithms" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "After" echo "" @@ -62,6 +67,7 @@ fi if ! [[ $quiet ]]; then print_sshd_t "macs" "Before"; fi sshd -T | grep "^macs" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "macs" "After" echo "" @@ -73,6 +79,7 @@ fi if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "Before"; fi sshd -T | grep "^hostkeyalgorithms" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "After" echo "" @@ -90,4 +97,3 @@ fi {% if grains['os'] != 'CentOS' %} print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." {% endif %} - diff --git a/setup/so-functions b/setup/so-functions index 29a58e718..deeefb3ba 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1436,8 +1436,6 @@ install_cleanup() { info "Removing so-setup permission entry from sudoers file" sed -i '/so-setup/d' /etc/sudoers fi - - so-ssh-harden -q } import_registry_docker() { diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..ba76b303e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -888,6 +888,7 @@ set_redirect >> $setup_log 2>&1 set_progress_str 85 'Applying finishing touches' filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 + so-ssh-harden -q >> $setup_log 2>&1 if [[ $is_manager || $is_import ]]; then set_progress_str 87 'Adding user to SOC' From 5ade0b9f40c37dffe27d42c89a4afaddcac97f9a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 16:30:30 -0400 Subject: [PATCH 014/258] Implement customizable overview page --- salt/soc/files/soc/changes.json | 49 --------------------------------- salt/soc/files/soc/motd.md | 17 ++++++++++++ salt/soc/init.sls | 8 +++--- 3 files changed, 21 insertions(+), 53 deletions(-) delete mode 100644 salt/soc/files/soc/changes.json create mode 100644 salt/soc/files/soc/motd.md diff --git a/salt/soc/files/soc/changes.json b/salt/soc/files/soc/changes.json deleted file mode 100644 index dbc7b4061..000000000 --- a/salt/soc/files/soc/changes.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "title": "Security Onion 2.3.40 is here!", - "changes": [ - { "summary": "FEATURE: Add option for HTTP Method Specification/POST to Hunt/Alerts Actions #2904" }, - { "summary": "FEATURE: Add option to configure proxy for various tools used during setup + persist the proxy configuration #529" }, - { "summary": "FEATURE: Alerts/Hunt - Provide method for base64-encoding pivot value #1749" }, - { "summary": "FEATURE: Allow users to customize links in SOC #1248" }, - { "summary": "FEATURE: Display user who requested PCAP in SOC #2775" }, - { "summary": "FEATURE: Make SOC browser app connection timeouts adjustable #2408" }, - { "summary": "FEATURE: Move to FleetDM #3483" }, - { "summary": "FEATURE: Reduce field cache expiration from 1d to 5m, and expose value as a salt pillar #3537" }, - { "summary": "FEATURE: Refactor docker_clean salt state to use loop w/ inspection instead of hardcoded image list #3113" }, - { "summary": "FEATURE: Run so-ssh-harden during setup #1932" }, - { "summary": "FEATURE: SOC should only display links to tools that are enabled #1643" }, - { "summary": "FEATURE: Update Sigmac Osquery Field Mappings #3137" }, - { "summary": "FEATURE: User must accept the Elastic licence during setup #3233" }, - { "summary": "FEATURE: soup should output more guidance for distributed deployments at the end #3340" }, - { "summary": "FEATURE: soup should provide some initial information and then prompt the user to continue #3486" }, - { "summary": "FIX: Add cronjob for so-suricata-eve-clean script #3515" }, - { "summary": "FIX: Change Elasticsearch heap formula #1686" }, - { "summary": "FIX: Create a post install version loop in soup #3102" }, - { "summary": "FIX: Custom Kibana settings are not being applied properly on upgrades #3254" }, - { "summary": "FIX: Hunt query issues with quotes #3320" }, - { "summary": "FIX: IP Addresses don't work with .security #3327" }, - { "summary": "FIX: Improve DHCP leases query in Hunt #3395" }, - { "summary": "FIX: Improve Setup verbiage #3422" }, - { "summary": "FIX: Improve Suricata DHCP logging and parsing #3397" }, - { "summary": "FIX: Keep RELATED,ESTABLISHED rules at the top of iptables chains #3288" }, - { "summary": "FIX: Populate http.status_message field #3408" }, - { "summary": "FIX: Remove 'types removal' deprecation messages from elastic log. #3345" }, - { "summary": "FIX: Reword + fix formatting on ES data storage prompt #3205" }, - { "summary": "FIX: SMTP shoud read SNMP on Kibana SNMP view #3413" }, - { "summary": "FIX: Sensors can temporarily show offline while processing large PCAP jobs #3279" }, - { "summary": "FIX: Soup should log to the screen as well as to a file #3467" }, - { "summary": "FIX: Strelka port 57314 not immediately relinquished upon restart #3457" }, - { "summary": "FIX: Switch SOC to pull from fieldcaps API due to field caching changes in Kibana 7.11 #3502" }, - { "summary": "FIX: Syntax error in /etc/sysctl.d/99-reserved-ports.conf #3308" }, - { "summary": "FIX: Telegraf hardcoded to use https and is not aware of elasticsearch features #2061" }, - { "summary": "FIX: Zeek Index Close and Delete Count for curator #3274" }, - { "summary": "FIX: so-cortex-user-add and so-cortex-user-enable use wrong pillar value for api key #3388" }, - { "summary": "FIX: so-rule does not completely apply change #3289" }, - { "summary": "FIX: soup should recheck disk space after it tries to clean up. #3235" }, - { "summary": "UPGRADE: Elastic 7.11.2 #3389" }, - { "summary": "UPGRADE: Suricata 6.0.2 #3217" }, - { "summary": "UPGRADE: Zeek 4 #3216" }, - { "summary": "UPGRADE: Zeek container to use Python 3 #1113" }, - { "summary": "UPGRADE: docker-ce to latest #3493" } - ] -} \ No newline at end of file diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md new file mode 100644 index 000000000..54df73d1b --- /dev/null +++ b/salt/soc/files/soc/motd.md @@ -0,0 +1,17 @@ +## Getting Started + +New to Security Onion 2? Check out the [Online Help](/docs/) and [Cheatsheet](/docs/cheatsheet.pdf) to learn how to best utilize Security Onion to hunt for evil! Find them in the upper-right menu. + +If you're ready to dive-in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. Or navigate to the [Hunt](/#/hunt) interface to hunt for evil that the alerts might have missed! + +## What's New + +The release notes have moved to the upper-right menu. Click on the [What's New](/docs/release-notes.html) menu option to find all the latest fixes and features in this version of Security Onion! + +## Customize This Space + +Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. + +To customize this content, copy `/opt/so/saltstack/default/salt/soc/files/soc/motd.md` to `/opt/so/saltstack/local/salt/soc/files/soc/motd.md` and edit it as desired. + +Visit [mardownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. \ No newline at end of file diff --git a/salt/soc/init.sls b/salt/soc/init.sls index d31898e72..a2d3ecf89 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -35,10 +35,10 @@ socconfig: - mode: 600 - template: jinja -socchanges: +socmotd: file.managed: - - name: /opt/so/conf/soc/changes.json - - source: salt://soc/files/soc/changes.json + - name: /opt/so/conf/soc/motd.md + - source: salt://soc/files/soc/motd.md - user: 939 - group: 939 - mode: 600 @@ -61,7 +61,7 @@ so-soc: - binds: - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - - /opt/so/conf/soc/changes.json:/opt/sensoroni/html/changes.json:ro + - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} From 887920e7c53ae66524ab682c6d10e6ffb9284809 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 16:44:08 -0400 Subject: [PATCH 015/258] Implement customizable overview page --- salt/soc/files/soc/motd.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index 54df73d1b..43da6732a 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -10,8 +10,16 @@ The release notes have moved to the upper-right menu. Click on the [What's New]( ## Customize This Space -Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. +Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. Visit [mardownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. -To customize this content, copy `/opt/so/saltstack/default/salt/soc/files/soc/motd.md` to `/opt/so/saltstack/local/salt/soc/files/soc/motd.md` and edit it as desired. +To customize this content, login to the manager via SSH and execute the following command: -Visit [mardownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. \ No newline at end of file +```bash +cp -f /opt/so/saltstack/default/salt/soc/files/soc/motd.md /opt/so/saltstack/local/salt/soc/files/soc/motd.md +``` + +Now, edit the new file as desired. Finally, run this command: + +```bash +salt-call state.apply soc queue=True +``` From 79ad87f83c4819353ba16bdf82f31ee472a83a2f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 21:16:17 -0400 Subject: [PATCH 016/258] Remove freqserver, minio, and domainstats from image list --- salt/common/tools/sbin/so-image-common | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 402ae97f3..be5a327f0 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -47,20 +47,17 @@ container_list() { TRUSTED_CONTAINERS=( "so-acng" "so-curator" - "so-domainstats" "so-elastalert" "so-elasticsearch" "so-filebeat" "so-fleet" "so-fleet-launcher" - "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" "so-kibana" "so-kratos" "so-logstash" - "so-minio" "so-mysql" "so-nginx" "so-pcaptools" From 982f2de33ce0e6025b49552345eb174f15652bc0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 24 Mar 2021 09:48:00 -0400 Subject: [PATCH 017/258] [fix] Refactor so-ssh-harden * Create a temp file to make changes, and only copy back over if any changes are made * Test changes as they're made, and exit if the test fails * Only add lines if they don't already exist in the config --- salt/common/tools/sbin/so-ssh-harden | 152 +++++++++++++++++---------- 1 file changed, 98 insertions(+), 54 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 5891e4309..0e0b28b39 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -6,14 +6,16 @@ if [[ $1 =~ ^(-q|--quiet) ]]; then quiet=true fi +sshd_config=/etc/ssh/sshd_config +temp_config=/tmp/sshd_config + before= after= reload_required=false -print_sshd_t() { +check_sshd_t() { local string=$1 local state=$2 - echo "${state}:" local grep_out grep_out=$(sshd -T | grep "^${string}") @@ -23,8 +25,17 @@ print_sshd_t() { else after=$grep_out fi +} - echo $grep_out +print_diff() { + local type=$1 + local diff + diff=$(diff -dqbB <(echo $before) <(echo $after)) + + if [[ -n $diff ]]; then + printf '%s\n' "$type" "$diff" + echo "" + fi } print_msg() { @@ -33,67 +44,100 @@ print_msg() { printf "%s\n" \ "----" \ "$msg" \ - "----" \ - "" + "----" fi } -# Add newline to ssh for legibility -echo "" >> /etc/ssh/sshd_config +add_if_missing() { + local string=$1 + if ! grep -q "$1" $temp_config; then + printf "%s\n\n" "$1" >> $temp_config + reload_required=true + fi +} -if ! [[ $quiet ]]; then print_sshd_t "ciphers" "Before"; fi -sshd -T | grep "^ciphers" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "ciphers" "After" - echo "" -fi +test_config() { + local msg + msg=$(sshd -t -f $temp_config) + local ret=$? -if [[ $before != $after ]]; then - reload_required=true -fi + if [[ -n $msg ]]; then + echo "Error found in temp sshd config:" + echo $msg + fi -if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "Before"; fi -sshd -T | grep "^kexalgorithms" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "kexalgorithms" "After" - echo "" -fi + return $ret +} -if [[ $before != $after ]]; then - reload_required=true -fi +main() { + if ! [[ $quiet ]]; then echo "Copying current config to $temp_config"; fi + cp $sshd_config $temp_config -if ! [[ $quiet ]]; then print_sshd_t "macs" "Before"; fi -sshd -T | grep "^macs" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "macs" "After" - echo "" -fi + # Add newline to ssh for legibility + echo "" >> $temp_config -if [[ $before != $after ]]; then - reload_required=true -fi + # Ciphers + check_sshd_t "ciphers" "Before" + local cipher_string + cipher_string=$(echo "$before" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g") -if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "Before"; fi -sshd -T | grep "^hostkeyalgorithms" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "hostkeyalgorithms" "After" - echo "" -fi + check_sshd_t "ciphers" "After" -if [[ $before != $after ]]; then - reload_required=true -fi + if ! [[ $quiet ]]; then print_diff "ciphers"; fi -if [[ $reload_required == true ]]; then - print_msg "Reloading sshd to load config changes..." - systemctl reload sshd -fi + if [[ $before != $after ]]; then + add_if_missing "$cipher_string" && test_config || exit 1 + fi -{% if grains['os'] != 'CentOS' %} -print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." -{% endif %} + # KexAlgorithms + check_sshd_t "kexalgorithms" "Before" + + local kexalg_string + kexalg_string=$(echo "$before" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g") + + check_sshd_t "kexalgorithms" "After" + + if ! [[ $quiet ]]; then print_diff "kexalgorithms"; fi + + if [[ $before != $after ]]; then + add_if_missing "$kexalg_string" && test_config || exit 1 + fi + + # Macs + check_sshd_t "macs" "Before" + local macs_string + macs_string=$(echo "$before" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g") + + check_sshd_t "macs" "After" + + if ! [[ $quiet ]]; then print_diff "macs"; fi + + if [[ $before != $after ]]; then + add_if_missing "$mac_string" && test_config || exit 1 + fi + + # HostKeyAlgorithms + check_sshd_t "hostkeyalgorithms" "Before" + local hostkeyalg_string + hostkeyalg_string=$(echo "$before" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g") + + check_sshd_t "hostkeyalgorithms" "After" + + if ! [[ $quiet ]]; then print_diff "hostkeyalgorithms"; fi + + if [[ $before != $after ]]; then + add_if_missing "$hostkeyalg_string" && test_config || exit 1 + fi + + if [[ $reload_required == true ]]; then + mv -f $temp_config $sshd_config + if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes..."; fi + systemctl reload sshd + print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." + else + if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up."; fi + rm -f $temp_config + fi +} + +main From 08f46a779ac6b1ae2f5754b078a7238f4eb3b9aa Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 21:16:17 -0400 Subject: [PATCH 018/258] Remove freqserver, minio, and domainstats from image list --- salt/common/tools/sbin/so-image-common | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 402ae97f3..be5a327f0 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -47,20 +47,17 @@ container_list() { TRUSTED_CONTAINERS=( "so-acng" "so-curator" - "so-domainstats" "so-elastalert" "so-elasticsearch" "so-filebeat" "so-fleet" "so-fleet-launcher" - "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" "so-kibana" "so-kratos" "so-logstash" - "so-minio" "so-mysql" "so-nginx" "so-pcaptools" From 7fc2467951c864627c0c3beff04b45259eb61f32 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 24 Mar 2021 15:00:02 -0400 Subject: [PATCH 019/258] Correct local online docs link to release notes --- salt/soc/files/soc/motd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index 43da6732a..295329f39 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -6,7 +6,7 @@ If you're ready to dive-in, take a look at the [Alerts](/#/alerts) interface to ## What's New -The release notes have moved to the upper-right menu. Click on the [What's New](/docs/release-notes.html) menu option to find all the latest fixes and features in this version of Security Onion! +The release notes have moved to the upper-right menu. Click on the [What's New](/docs/#document-release-notes) menu option to find all the latest fixes and features in this version of Security Onion! ## Customize This Space From af3951e1ad7dff3f28f03dbeb10466facf8d3cbb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Mar 2021 11:51:55 -0400 Subject: [PATCH 020/258] Attempt to use so repo for network install --- setup/so-functions | 49 ++++++++++++++++---------- setup/so-setup | 2 ++ setup/yum_repos/saltstack.repo | 6 ---- setup/yum_repos/securityonion.repo | 56 ++++++++++++++++++++++++++++++ setup/yum_repos/wazuh.repo | 7 ---- 5 files changed, 88 insertions(+), 32 deletions(-) delete mode 100644 setup/yum_repos/saltstack.repo create mode 100644 setup/yum_repos/securityonion.repo delete mode 100644 setup/yum_repos/wazuh.repo diff --git a/setup/so-functions b/setup/so-functions index 29a58e718..b6cf569fb 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1085,9 +1085,9 @@ docker_install() { if [ $OS = 'centos' ]; then { yum clean expire-cache; - if [[ ! $is_airgap ]]; then - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; - fi + #if [[ ! $is_airgap ]]; then + # yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; + #fi if [[ ! $is_iso ]]; then yum -y install docker-ce-20.10.5-3.el7 containerd.io-1.4.4-3.1.el7; fi @@ -1990,8 +1990,8 @@ saltify() { if [ $OS = 'centos' ]; then set_progress_str 5 'Installing Salt repo' { - sudo rpm --import https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/SALTSTACK-GPG-KEY.pub; - cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; + sudo rpm --import https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub; + #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; } >> "$setup_log" 2>&1 set_progress_str 6 'Installing various dependencies' if [[ ! $is_iso ]]; then @@ -2001,7 +2001,7 @@ saltify() { 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORT') reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then - logCmd "yum -y install epel-release" + #logCmd "yum -y install epel-release" logCmd "yum -y install sqlite argon2 curl mariadb-devel" fi # Download Ubuntu Keys in case manager updates = 1 @@ -2010,7 +2010,7 @@ saltify() { logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" logCmd "wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH" - logCmd "cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo" + #logCmd "cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo" fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then @@ -2019,29 +2019,29 @@ saltify() { systemctl enable salt-master >> "$setup_log" 2>&1 ;; *) - if [ "$MANAGERUPDATES" = '1' ]; then - { - if [[ ! $is_airgap ]]; then + #if [ "$MANAGERUPDATES" = '1' ]; then + # { + # if [[ ! $is_airgap ]]; then # Create the GPG Public Key for the Salt Repo - cp ./public_keys/salt.pem /etc/pki/rpm-gpg/saltstack-signing-key; + #cp ./public_keys/salt.pem /etc/pki/rpm-gpg/saltstack-signing-key; # Copy repo files over - cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; - else - info "This is airgap" - fi - } >> "$setup_log" 2>&1 - fi + #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; + # else + # info "This is airgap" + # fi + # } >> "$setup_log" 2>&1 + #fi ;; esac if [[ ! $is_airgap ]]; then - cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo >> "$setup_log" 2>&1 + #cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo >> "$setup_log" 2>&1 yum clean expire-cache >> "$setup_log" 2>&1 fi set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - yum -y install epel-release + #yum -y install epel-release yum -y install salt-minion-3002.5\ python3\ python36-docker\ @@ -2264,6 +2264,17 @@ secrets_pillar(){ fi } +securityonion_repo() { + # Remove all the current repos + if [ "$OS" = 'centos' ]; then + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + else + echo "This is Ubuntu" + fi +} + set_base_heapsizes() { es_heapsize ls_heapsize diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..a532158f4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -622,6 +622,8 @@ set_redirect >> $setup_log 2>&1 info "Creating airgap repo" create_repo >> $setup_log 2>&1 airgap_rules >> $setup_log 2>&1 + else + securityonion_repo >> $setup_log 2>&1 fi if [[ $is_minion ]]; then diff --git a/setup/yum_repos/saltstack.repo b/setup/yum_repos/saltstack.repo deleted file mode 100644 index 0430a62b8..000000000 --- a/setup/yum_repos/saltstack.repo +++ /dev/null @@ -1,6 +0,0 @@ -[saltstack] -name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/ -enabled=1 -gpgcheck=1 -gpgkey=https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/SALTSTACK-GPG-KEY.pub \ No newline at end of file diff --git a/setup/yum_repos/securityonion.repo b/setup/yum_repos/securityonion.repo new file mode 100644 index 000000000..20c907289 --- /dev/null +++ b/setup/yum_repos/securityonion.repo @@ -0,0 +1,56 @@ +[base] +name=CentOS-$releasever - Base +baseurl=https://repo.securityonion.net/file/securityonion-repo/base/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#released updates +[updates] +name=CentOS-$releasever - Updates +baseurl=https://repo.securityonion.net/file/securityonion-repo/updates/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that may be useful +[extras] +name=CentOS-$releasever - Extras +baseurl=https://repo.securityonion.net/file/securityonion-repo/extras/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that extend functionality of existing packages +[centosplus] +name=CentOS-$releasever - Plus +baseurl=https://repo.securityonion.net/file/securityonion-repo/centosplus/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +[epel] +name=Extra Packages for Enterprise Linux 7 - $basearch +baseurl=https://repo.securityonion.net/file/securityonion-repo/epel/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/RPM-GPG-KEY-EPEL-7 + +[docker-ce-stable] +name=Docker CE Stable - $basearch +baseurl=https://repo.securityonion.net/file/securityonion-repo/docker-ce-stable +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub + +[saltstack] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + +[wazuh_repo] +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 \ No newline at end of file diff --git a/setup/yum_repos/wazuh.repo b/setup/yum_repos/wazuh.repo deleted file mode 100644 index ae462c62f..000000000 --- a/setup/yum_repos/wazuh.repo +++ /dev/null @@ -1,7 +0,0 @@ -[wazuh_repo] -gpgcheck=1 -gpgkey=https://packages.wazuh.com/key/GPG-KEY-WAZUH -enabled=1 -name=Wazuh repository -baseurl=https://packages.wazuh.com/3.x/yum/ -protect=1 From 150e724a4a52f0e58a224f3d1f82311964118abb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 25 Mar 2021 13:37:54 -0400 Subject: [PATCH 021/258] Fix chrony install logic + add sleep for chrony to finish sync --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index ffaa079c1..c2ddb2125 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -717,7 +717,7 @@ configure_ntp() { local chrony_conf=/etc/chrony.conf # Install chrony if it isn't already installed - if command -v chronyc &> /dev/null; then + if ! command -v chronyc &> /dev/null; then if [ "$OS" == centos ]; then yum -y install chrony else @@ -743,6 +743,7 @@ configure_ntp() { # Sync time & update the system time chronyc -a 'burst 4/4' + sleep 20 # Wait for chrony to sync chronyc -a makestep } From eb674b3b938b3769ab5ae0886b80d7a6a462c29d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 25 Mar 2021 14:45:33 -0400 Subject: [PATCH 022/258] Validate list of ntp servers (ip4, hostname, or fqdn) --- salt/common/tools/sbin/so-common | 14 ++++++++++++++ setup/so-functions | 13 ++++++++++--- setup/so-setup | 2 +- setup/so-variables | 4 ++-- setup/so-whiptail | 2 -- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 676b908ce..340525272 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -419,6 +419,20 @@ valid_proxy() { [[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1 } +valid_ntp_list() { + local string=$1 + local ntp_arr + IFS="," read -r -a ntp_arr <<< "$string" + + for ntp in "${ntp_arr[@]}"; do + if ! valid_ip4 "$ntp" && ! valid_hostname "$ntp" && ! valid_fqdn "$ntp"; then + return 1 + fi + done + + return 0 +} + valid_string() { local str=$1 local min_length=${2:-1} diff --git a/setup/so-functions b/setup/so-functions index c2ddb2125..6dd10096b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -489,10 +489,17 @@ collect_node_ls_pipeline_worker_count() { collect_ntp_servers() { if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then if whiptail_ntp_ask; then - [[ $is_airgap ]] && ntp_servers="" - whiptail_ntp_servers "$ntp_servers" + [[ $is_airgap ]] && ntp_servers=() + whiptail_ntp_servers "$ntp_string" + + while ! valid_ntp_list "$ntp_string"; do + whiptail_invalid_input + whiptail_ntp_servers "$ntp_string" + done + + IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array else - ntp_servers="" + ntp_servers=() fi fi } diff --git a/setup/so-setup b/setup/so-setup index 07eb49500..d9e64105a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -584,7 +584,7 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' - [[ -z $ntp_configured ]] && [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 + [[ -z $ntp_configured ]] && [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 reserve_ports diff --git a/setup/so-variables b/setup/so-variables index 0a07fc79d..676cba4f0 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -73,5 +73,5 @@ export install_opt_file net_init_file=/root/net_init export net_init_file -ntp_servers="0.pool.ntp.org,1.pool.ntp.org" -export ntp_servers +ntp_string="0.pool.ntp.org,1.pool.ntp.org" +export ntp_string diff --git a/setup/so-whiptail b/setup/so-whiptail index 2743ab65b..00397a6fa 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1119,8 +1119,6 @@ whiptail_ntp_servers() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - - IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array } whiptail_oinkcode() { From 0195d366cc9b386f73e1aa47ce4c437fb354118b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 26 Mar 2021 14:44:27 -0400 Subject: [PATCH 023/258] Add custom banner to login page --- salt/nginx/etc/nginx.conf | 2 +- salt/soc/files/soc/banner.md | 0 salt/soc/init.sls | 10 ++++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 salt/soc/files/soc/banner.md diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 25e8bc11f..ea820442b 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -157,7 +157,7 @@ http { ssl_prefer_server_ciphers on; ssl_protocols TLSv1.2; - location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) { + location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*) { proxy_pass http://{{ manager_ip }}:9822; proxy_read_timeout 90; proxy_connect_timeout 90; diff --git a/salt/soc/files/soc/banner.md b/salt/soc/files/soc/banner.md new file mode 100644 index 000000000..e69de29bb diff --git a/salt/soc/init.sls b/salt/soc/init.sls index a2d3ecf89..18fda41da 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -44,6 +44,15 @@ socmotd: - mode: 600 - template: jinja +socbanner: + file.managed: + - name: /opt/so/conf/soc/banner.md + - source: salt://soc/files/soc/banner.md + - user: 939 + - group: 939 + - mode: 600 + - template: jinja + soccustom: file.managed: - name: /opt/so/conf/soc/custom.js @@ -62,6 +71,7 @@ so-soc: - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro + - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} From 8819cc1371ae9106e323f39c67cbe9b071d12d64 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:01:22 -0400 Subject: [PATCH 024/258] Fix acng to actually cache --- salt/airgap/init.sls | 2 +- salt/common/keys/GPG-KEY-WAZUH | 52 +++++++++++++++++++++++ salt/common/keys/RPM-GPG-KEY-EPEL-7 | 29 +++++++++++++ salt/common/keys/SALTSTACK-GPG-KEY.pub | 31 ++++++++++++++ salt/common/keys/docker.pub | 28 +++++++++++++ salt/common/keys/securityonion.pub | 52 +++++++++++++++++++++++ salt/common/tools/sbin/so-common | 17 ++++++++ salt/common/tools/sbin/soup | 1 + salt/manager/files/acng/acng.conf | 1 + setup/so-functions | 51 +++------------------- setup/so-setup | 4 +- setup/yum_repos/securityonioncache.repo | 56 +++++++++++++++++++++++++ 12 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 salt/common/keys/GPG-KEY-WAZUH create mode 100644 salt/common/keys/RPM-GPG-KEY-EPEL-7 create mode 100644 salt/common/keys/SALTSTACK-GPG-KEY.pub create mode 100644 salt/common/keys/docker.pub create mode 100644 salt/common/keys/securityonion.pub create mode 100644 setup/yum_repos/securityonioncache.repo diff --git a/salt/airgap/init.sls b/salt/airgap/init.sls index 818bb3a3b..4ff401099 100644 --- a/salt/airgap/init.sls +++ b/salt/airgap/init.sls @@ -11,7 +11,7 @@ airgap_repo: pkgrepo.managed: - humanname: Airgap Repo - baseurl: https://{{ MANAGER }}/repo - - gpgcheck: 0 + - gpgcheck: 1 - sslverify: 0 agbase: diff --git a/salt/common/keys/GPG-KEY-WAZUH b/salt/common/keys/GPG-KEY-WAZUH new file mode 100644 index 000000000..b424ccfae --- /dev/null +++ b/salt/common/keys/GPG-KEY-WAZUH @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFeeyYwBEACyf4VwV8c2++J5BmCl6ofLCtSIW3UoVrF4F+P19k/0ngnSfjWb +8pSWB11HjZ3Mr4YQeiD7yY06UZkrCXk+KXDlUjMK3VOY7oNPkqzNaP6+8bDwj4UA +hADMkaXBvWooGizhCoBtDb1bSbHKcAnQ3PTdiuaqF5bcyKk8hv939CHulL2xH+BP +mmTBi+PM83pwvR+VRTOT7QSzf29lW1jD79v4rtXHJs4KCz/amT/nUm/tBpv3q0sT +9M9rH7MTQPdqvzMl122JcZST75GzFJFl0XdSHd5PAh2mV8qYak5NYNnwA41UQVIa ++xqhSu44liSeZWUfRdhrQ/Nb01KV8lLAs11Sz787xkdF4ad25V/Rtg/s4UXt35K3 +klGOBwDnzPgHK/OK2PescI5Ve1z4x1C2bkGze+gk/3IcfGJwKZDfKzTtqkZ0MgpN +7RGghjkH4wpFmuswFFZRyV+s7jXYpxAesElDSmPJ0O07O4lQXQMROE+a2OCcm0eF +3+Cr6qxGtOp1oYMOVH0vOLYTpwOkAM12/qm7/fYuVPBQtVpTojjV5GDl2uGq7p0o +h9hyWnLeNRbAha0px6rXcF9wLwU5n7mH75mq5clps3sP1q1/VtP/Fr84Lm7OGke4 +9eD+tPNCdRx78RNWzhkdQxHk/b22LCn1v6p1Q0qBco9vw6eawEkz1qwAjQARAQAB +tDFXYXp1aC5jb20gKFdhenVoIFNpZ25pbmcgS2V5KSA8c3VwcG9ydEB3YXp1aC5j +b20+iQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheABQJZHNOBBQkU +SgzvAAoJEJaz7l8pERFF6xUP/3SbcmrI/u7a2EqZ0GxwQ/LRkPzWkJRnozCtNYHD +ZjiZgSB/+77hkPS0tsBK/GXFLKfJAuf13XFrCvEuI4Q/pLOCCKIGumKXItUIwJBD +HiEmVt/XxIijmlF7O1jcWqE/5CQXofjr03WMx+qzNabIwU/6dTKZN4FrR1jDk7yS +6FYBsbhVcSoqSpGYx7EcuK3c3sKKtnbacK2Sw3K9n8Wdj+EK83cbpMg8D/efVRqv +xypeCeojtY10y4bmugEwMYPgFkrSbicuiZc8NA8qhvFp6JFRq/uL0PGACyg05wB3 +S9U4wvSkmlo2/G74awna22UlaoYmSSz3UZdpWd2zBxflx17948QfTqyhO6bM8qLz +dSyR6/6olAcR1N+PBup8PoMdBte4ul/hJp8WIviW0AxJUTZSbVj5v/t43QAKEpCE +IMHvkK8PRHz/9kMd/2xN7LgMtihCrGZOnzErkjhlZvmiJ6kcJoD7ywzFnfJrntOU +DjNb3eqUFSEwmhD60Hd2OCkfmiV7NEE/YTd9B72NSwzj4Za/JUdlF64LMeIiHbYp +Lh7P+mR+lMJf/SWsQmlyuiQ2u8SY2aDFvzBS9WtpwiznuUdrbRN87+TYLSVqDifj +Ea3zOnzLaLYbOr6LHz1xbhAvInv7KLobgiw1E4WnBNWN8xVwVJLKNE7wV88k43XV +3L/RuQINBFeeyYwBEADD1Y3zW5OrnYZ6ghTd5PXDAMB8Z1ienmnb2IUzLM+i0yE2 +TpKSP/XYCTBhFa390rYgFO2lbLDVsiz7Txd94nHrdWXGEQfwrbxsvdlLLWk7iN8l +Fb4B60OfRi3yoR96a/kIPNa0x26+n79LtDuWZ/DTq5JSHztdd9F1sr3h8i5zYmtv +luj99ZorpwYejbBVUm0+gP0ioaXM37uO56UFVQk3po9GaS+GtLnlgoE5volgNYyO +rkeIua4uZVsifREkHCKoLJip6P7S3kTyfrpiSLhouEZ7kV1lbMbFgvHXyjm+/AIx +HIBy+H+e+HNt5gZzTKUJsuBjx44+4jYsOR67EjOdtPOpgiuJXhedzShEO6rbu/O4 +wM1rX45ZXDYa2FGblHCQ/VaS0ttFtztk91xwlWvjTR8vGvp5tIfCi+1GixPRQpbN +Y/oq8Kv4A7vB3JlJscJCljvRgaX0gTBzlaF6Gq0FdcWEl5F1zvsWCSc/Fv5WrUPY +5mG0m69YUTeVO6cZS1aiu9Qh3QAT/7NbUuGXIaAxKnu+kkjLSz+nTTlOyvbG7BVF +a6sDmv48Wqicebkc/rCtO4g8lO7KoA2xC/K/6PAxDrLkVyw8WPsAendmezNfHU+V +32pvWoQoQqu8ysoaEYc/j9fN4H3mEBCN3QUJYCugmHP0pu7VtpWwwMUqcGeUVwAR +AQABiQIlBBgBCAAPAhsMBQJZHNOaBQkUSg0HAAoJEJaz7l8pERFFhpkQAJ09mjjp +n9f18JGSMzP41fVucPuLBZ5XJL/hy2boII1FvgfmOETzNxLPblHdkJVjZS5iMrhL +EJ1jv+GQDtf68/0jO+HXuQIBmUJ53YwbuuQlLWH7CI2AxlSAKAn2kOApWMKsjnAv +JwS3eNGukOKWRfEKTqz2Vwi1H7M7ppypZ9keoyAoSIWb61gm7rXbfT+tVBetHfrU +EM5vz3AS3pJk6Yfqn10IZfiexXmsBD+SpJBNzMBsznCcWO2y4qZNLjFferBoizvV +34UnZyd1bkSN0T/MKp8sgJwqDJBS72tH6ZIM8NNoy29aPDkeaa8XlhkWiBdRizqL +BcxrV/1n3xdzfY9FX6s4KGudo+gYsVpY0mrpZU8jG8YUNLDXQTXnRo4CQOtRJJbA +RFDoZfsDqToZftuEhIsk+MaKlyXoA0eIYqGe6lXa/jEwvViqLYubCNLu0+kgNQ3v +hKF8Pf7eXFDAePw7guuvDvBOMQqBCaKCxsz1HoKRNYBEdUYrEQBJnX235Q4IsdI/ +GcQ/dvERJXaDCG8EPhnwc517EMUJDiJ1CxT4+VMHphmFbiVqmctz0upIj+D037Xk +CcgxNte6LZorGRZ/l1MYINliGJKtCCFK7XGVPKiJ8zyGSyPj1FfwtBy5hUX3aQtm +bvP0H2BRCKoelsbRENu58BkU6YhiUry7pVul +=SJij +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/RPM-GPG-KEY-EPEL-7 b/salt/common/keys/RPM-GPG-KEY-EPEL-7 new file mode 100644 index 000000000..f205ede46 --- /dev/null +++ b/salt/common/keys/RPM-GPG-KEY-EPEL-7 @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.11 (GNU/Linux) + +mQINBFKuaIQBEAC1UphXwMqCAarPUH/ZsOFslabeTVO2pDk5YnO96f+rgZB7xArB +OSeQk7B90iqSJ85/c72OAn4OXYvT63gfCeXpJs5M7emXkPsNQWWSju99lW+AqSNm +jYWhmRlLRGl0OO7gIwj776dIXvcMNFlzSPj00N2xAqjMbjlnV2n2abAE5gq6VpqP +vFXVyfrVa/ualogDVmf6h2t4Rdpifq8qTHsHFU3xpCz+T6/dGWKGQ42ZQfTaLnDM +jToAsmY0AyevkIbX6iZVtzGvanYpPcWW4X0RDPcpqfFNZk643xI4lsZ+Y2Er9Yu5 +S/8x0ly+tmmIokaE0wwbdUu740YTZjCesroYWiRg5zuQ2xfKxJoV5E+Eh+tYwGDJ +n6HfWhRgnudRRwvuJ45ztYVtKulKw8QQpd2STWrcQQDJaRWmnMooX/PATTjCBExB +9dkz38Druvk7IkHMtsIqlkAOQMdsX1d3Tov6BE2XDjIG0zFxLduJGbVwc/6rIc95 +T055j36Ez0HrjxdpTGOOHxRqMK5m9flFbaxxtDnS7w77WqzW7HjFrD0VeTx2vnjj +GqchHEQpfDpFOzb8LTFhgYidyRNUflQY35WLOzLNV+pV3eQ3Jg11UFwelSNLqfQf +uFRGc+zcwkNjHh5yPvm9odR1BIfqJ6sKGPGbtPNXo7ERMRypWyRz0zi0twARAQAB +tChGZWRvcmEgRVBFTCAoNykgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQI4BBMB +AgAiBQJSrmiEAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBqL66iNSxk +5cfGD/4spqpsTjtDM7qpytKLHKruZtvuWiqt5RfvT9ww9GUUFMZ4ZZGX4nUXg49q +ixDLayWR8ddG/s5kyOi3C0uX/6inzaYyRg+Bh70brqKUK14F1BrrPi29eaKfG+Gu +MFtXdBG2a7OtPmw3yuKmq9Epv6B0mP6E5KSdvSRSqJWtGcA6wRS/wDzXJENHp5re +9Ism3CYydpy0GLRA5wo4fPB5uLdUhLEUDvh2KK//fMjja3o0L+SNz8N0aDZyn5Ax +CU9RB3EHcTecFgoy5umRj99BZrebR1NO+4gBrivIfdvD4fJNfNBHXwhSH9ACGCNv +HnXVjHQF9iHWApKkRIeh8Fr2n5dtfJEF7SEX8GbX7FbsWo29kXMrVgNqHNyDnfAB +VoPubgQdtJZJkVZAkaHrMu8AytwT62Q4eNqmJI1aWbZQNI5jWYqc6RKuCK6/F99q +thFT9gJO17+yRuL6Uv2/vgzVR1RGdwVLKwlUjGPAjYflpCQwWMAASxiv9uPyYPHc +ErSrbRG0wjIfAR3vus1OSOx3xZHZpXFfmQTsDP7zVROLzV98R3JwFAxJ4/xqeON4 +vCPFU6OsT3lWQ8w7il5ohY95wmujfr6lk89kEzJdOTzcn7DBbUru33CQMGKZ3Evt +RjsC7FDbL017qxS+ZVA/HGkyfiu4cpgV8VUnbql5eAZ+1Ll6Dw== +=hdPa +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/SALTSTACK-GPG-KEY.pub b/salt/common/keys/SALTSTACK-GPG-KEY.pub new file mode 100644 index 000000000..14bd7d98c --- /dev/null +++ b/salt/common/keys/SALTSTACK-GPG-KEY.pub @@ -0,0 +1,31 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2 + +mQENBFOpvpgBCADkP656H41i8fpplEEB8IeLhugyC2rTEwwSclb8tQNYtUiGdna9 +m38kb0OS2DDrEdtdQb2hWCnswxaAkUunb2qq18vd3dBvlnI+C4/xu5ksZZkRj+fW +tArNR18V+2jkwcG26m8AxIrT+m4M6/bgnSfHTBtT5adNfVcTHqiT1JtCbQcXmwVw +WbqS6v/LhcsBE//SHne4uBCK/GHxZHhQ5jz5h+3vWeV4gvxS3Xu6v1IlIpLDwUts +kT1DumfynYnnZmWTGc6SYyIFXTPJLtnoWDb9OBdWgZxXfHEcBsKGha+bXO+m2tHA +gNneN9i5f8oNxo5njrL8jkCckOpNpng18BKXABEBAAG0MlNhbHRTdGFjayBQYWNr +YWdpbmcgVGVhbSA8cGFja2FnaW5nQHNhbHRzdGFjay5jb20+iQE4BBMBAgAiBQJT +qb6YAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAOCKFJ3le/vhkqB/0Q +WzELZf4d87WApzolLG+zpsJKtt/ueXL1W1KA7JILhXB1uyvVORt8uA9FjmE083o1 +yE66wCya7V8hjNn2lkLXboOUd1UTErlRg1GYbIt++VPscTxHxwpjDGxDB1/fiX2o +nK5SEpuj4IeIPJVE/uLNAwZyfX8DArLVJ5h8lknwiHlQLGlnOu9ulEAejwAKt9CU +4oYTszYM4xrbtjB/fR+mPnYh2fBoQO4d/NQiejIEyd9IEEMd/03AJQBuMux62tjA +/NwvQ9eqNgLw9NisFNHRWtP4jhAOsshv1WW+zPzu3ozoO+lLHixUIz7fqRk38q8Q +9oNR31KvrkSNrFbA3D89uQENBFOpvpgBCADJ79iH10AfAfpTBEQwa6vzUI3Eltqb +9aZ0xbZV8V/8pnuU7rqM7Z+nJgldibFk4gFG2bHCG1C5aEH/FmcOMvTKDhJSFQUx +uhgxttMArXm2c22OSy1hpsnVG68G32Nag/QFEJ++3hNnbyGZpHnPiYgej3FrerQJ +zv456wIsxRDMvJ1NZQB3twoCqwapC6FJE2hukSdWB5yCYpWlZJXBKzlYz/gwD/Fr +GL578WrLhKw3UvnJmlpqQaDKwmV2s7MsoZogC6wkHE92kGPG2GmoRD3ALjmCvN1E +PsIsQGnwpcXsRpYVCoW7e2nW4wUf7IkFZ94yOCmUq6WreWI4NggRcFC5ABEBAAGJ +AR8EGAECAAkFAlOpvpgCGwwACgkQDgihSd5Xv74/NggA08kEdBkiWWwJZUZEy7cK +WWcgjnRuOHd4rPeT+vQbOWGu6x4bxuVf9aTiYkf7ZjVF2lPn97EXOEGFWPZeZbH4 +vdRFH9jMtP+rrLt6+3c9j0M8SIJYwBL1+CNpEC/BuHj/Ra/cmnG5ZNhYebm76h5f +T9iPW9fFww36FzFka4VPlvA4oB7ebBtquFg3sdQNU/MmTVV4jPFWXxh4oRDDR+8N +1bcPnbB11b5ary99F/mqr7RgQ+YFF0uKRE3SKa7a+6cIuHEZ7Za+zhPaQlzAOZlx +fuBmScum8uQTrEF5+Um5zkwC7EXTdH1co/+/V/fpOtxIg4XO4kcugZefVm5ERfVS +MA== +=dtMN +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/docker.pub b/salt/common/keys/docker.pub new file mode 100644 index 000000000..1967cbf01 --- /dev/null +++ b/salt/common/keys/docker.pub @@ -0,0 +1,28 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFit5IEBEADDt86QpYKz5flnCsOyZ/fk3WwBKxfDjwHf/GIflo+4GWAXS7wJ +1PSzPsvSDATV10J44i5WQzh99q+lZvFCVRFiNhRmlmcXG+rk1QmDh3fsCCj9Q/yP +w8jn3Hx0zDtz8PIB/18ReftYJzUo34COLiHn8WiY20uGCF2pjdPgfxE+K454c4G7 +gKFqVUFYgPug2CS0quaBB5b0rpFUdzTeI5RCStd27nHCpuSDCvRYAfdv+4Y1yiVh +KKdoe3Smj+RnXeVMgDxtH9FJibZ3DK7WnMN2yeob6VqXox+FvKYJCCLkbQgQmE50 +uVK0uN71A1mQDcTRKQ2q3fFGlMTqJbbzr3LwnCBE6hV0a36t+DABtZTmz5O69xdJ +WGdBeePCnWVqtDb/BdEYz7hPKskcZBarygCCe2Xi7sZieoFZuq6ltPoCsdfEdfbO ++VBVKJnExqNZCcFUTEnbH4CldWROOzMS8BGUlkGpa59Sl1t0QcmWlw1EbkeMQNrN +spdR8lobcdNS9bpAJQqSHRZh3cAM9mA3Yq/bssUS/P2quRXLjJ9mIv3dky9C3udM ++q2unvnbNpPtIUly76FJ3s8g8sHeOnmYcKqNGqHq2Q3kMdA2eIbI0MqfOIo2+Xk0 +rNt3ctq3g+cQiorcN3rdHPsTRSAcp+NCz1QF9TwXYtH1XV24A6QMO0+CZwARAQAB +tCtEb2NrZXIgUmVsZWFzZSAoQ0UgcnBtKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 +BBMBCgAhBQJYrep4AhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEMUv62ti +Hp816C0P/iP+1uhSa6Qq3TIc5sIFE5JHxOO6y0R97cUdAmCbEqBiJHUPNQDQaaRG +VYBm0K013Q1gcJeUJvS32gthmIvhkstw7KTodwOM8Kl11CCqZ07NPFef1b2SaJ7l +TYpyUsT9+e343ph+O4C1oUQw6flaAJe+8ATCmI/4KxfhIjD2a/Q1voR5tUIxfexC +/LZTx05gyf2mAgEWlRm/cGTStNfqDN1uoKMlV+WFuB1j2oTUuO1/dr8mL+FgZAM3 +ntWFo9gQCllNV9ahYOON2gkoZoNuPUnHsf4Bj6BQJnIXbAhMk9H2sZzwUi9bgObZ +XO8+OrP4D4B9kCAKqqaQqA+O46LzO2vhN74lm/Fy6PumHuviqDBdN+HgtRPMUuao +xnuVJSvBu9sPdgT/pR1N9u/KnfAnnLtR6g+fx4mWz+ts/riB/KRHzXd+44jGKZra +IhTMfniguMJNsyEOO0AN8Tqcl0eRBxcOArcri7xu8HFvvl+e+ILymu4buusbYEVL +GBkYP5YMmScfKn+jnDVN4mWoN1Bq2yMhMGx6PA3hOvzPNsUoYy2BwDxNZyflzuAi +g59mgJm2NXtzNbSRJbMamKpQ69mzLWGdFNsRd4aH7PT7uPAURaf7B5BVp3UyjERW +5alSGnBqsZmvlRnVH5BDUhYsWZMPRQS9rRr4iGW0l+TH+O2VJ8aQ +=0Zqq +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/securityonion.pub b/salt/common/keys/securityonion.pub new file mode 100644 index 000000000..c9148ff5c --- /dev/null +++ b/salt/common/keys/securityonion.pub @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF7rzwEBEADBg87uJhnC3Ls7s60hbHGaywGrPtbz2WuYA/ev3YS3X7WS75p8 +PGlzTWUCujx0pEHbK2vYfExl3zksZ8ZmLyZ9VB3oSLiWBzJgKAeB7YCFEo8te+eE +P2Z+8c+kX4eOV+2waxZyewA2TipSkhWgStSI4Ow8SyVUcUWA3hCw7mo2duNVi7KO +C3vvI3wzirH+8/XIGo+lWTg6yYlSxdf+0xWzYvV2QCMpwzJfARw6GGXtfCZw/zoO +o4+YPsiyztQdyI1y+g3Fbesl65E36DelbyP+lYd2VecX8ELEv0wlKCgHYlk6lc+n +qnOotVjWbsyXuFfo06PHUd6O9n3nmo0drC6kmXGw1e8hu0t8VcGfMTKS/hszwVUY +bHS6kbfsOoAb6LXPWKfqxk/BdreLXmcHHz88DimS3OS0JufkcmkjxEzSFRL0kb2h +QVb1SATrbx+v2RWQXvi9sLCjT2fdOiwi1Tgc84orc7A1C3Jwu353YaX9cV+n5uyG +OZ2AULZ5z2h13sVuiZAwfyyFs/O0CJ783hFA2TNPnyNGAgw/kaIo7nNRnggtndBo +oQzVS+BHiFx98IF4zDqmF2r2+jOCjxSrw8KnZBe4bgXFtl89DmjoejGvWDnu2MVM +pZDEs1DcOxHBQmTCWMIYLyNKG0xW6diyWBxEIaa7YgrP6kA+RaDfZ/xXPwARAQAB +tD9TZWN1cml0eSBPbmlvbiBTb2x1dGlvbnMsIExMQyA8aW5mb0BzZWN1cml0eW9u +aW9uc29sdXRpb25zLmNvbT6JAlQEEwEKAD4WIQTIBKk9Nr4Mcz6hlkR8EGC3/lBw +EwUCXuvPAQIbAwUJEswDAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRB8EGC3 +/lBwExB1D/42xIDGU2XFNFyTU+ZqzDA8qNC9hEKjLeizbeM8RIm3xO+3p7SdqbuJ +7pA8gk0RiHuILb+Ba1xiSh/w/W2bOxQhsXuWHih2z3W1tI+hu6RQhIm4e6CIHHf7 +Vzj4RSvHOVS0AzITUwkHjv0x0Z8zVBPJfEHKkK2x03BqP1o12rd7n2ZMrSfN6sED +fUwOJLDjthShtyLSPBVG8j7T5cfSCPSLhfVOKPQVcI1sSir7RLeyxt1v1kzjQdaA ++znxO8EgfZJN93wzfBrAGcVT8KmpmgwR6p46m20wJXyZC9DZxJ0o1y3toVWTC+kP +Qj1ROPivySVn10rBoOJk8HteyhW07gTcydq+noKHV7SqJ1899xRAYP7rDCfI9iMW +Nn22ZDLnAkIcbNR7JLJCHwsZH/Umo9KO/dIccIqVQel3UCCYZcWTZW0VkcjqVKRa +eK+JQGaJPrBAoxIG5/sMlbk2sINSubNWlcbH6kM0V8NVwdPiOO9xLmp2hI4ICxE3 +M+O2HCNX4QYzVizzTFxEvW3ieLa4nePQ8J6lvMI2oLkFP7xHoFluvZnuwfNvoEy0 +RnlHExN1UQTUvcbCxIbzjaJ4HJXilWHjgmGaVQO1S7AYskWnNWQ7uJvxnuZBNNwm +pIvwYEZp23fYaWl/xKqnmPMy2ADjROBKlCm7L+Ntq1r7ELGW5ZCTobkCDQRe688B +ARAA22GzdkSAo+mwJ2S1RbJ1G20tFnLsG/NC8iMN3lEh/PSmyPdB7mBtjZ+HPDzF +VSznXZdr3LItBBQOli2hVIj1lZBY7+s2ZufV3TFFwselUwT3b1g1KMkopD95Ckf8 +WhLbSz2yqgrvcEvbB0HFX/ZEsHGqIz2kLacixjwXXLWOMQ2LNbeW1f5zQkBnaNNQ +/4njzTj68OxnvfplNYNJqi2pZGb2UqarYX04FqKNuocN8E7AC9FQdBXylmVctw9T +pQVwfCI76bTe6vPWb+keb6UNN1jyXVnhIQ3Fv5sFBsmgXf/hO8tqCotrKjEiK2/i +RkvFeqsGMXreCgYg9zW4k+DcJtVa+Q8juGOjElrubY3Ua9mCusx3vY4QYSWxQ5Ih +k1lXiUcM5Rt38lfpKHRJ5Pd4Y5xlWSQfZ7nmzbf/GzJQz+rWrA0X6Oc6cDOPLNXK +w1dAygre4f2bsp5kHQt6NMefxeNTDmi+4R62K0tb40f5q0Vxz8qdyD48bBsbULNx +kb6mjOAD+FNkfNXcGeuTq9oRnjx8i93mhYsIP5LFNDXS/zSP1nv0ZUFeIlGQGjV9 +1wOvT454qkI9sKiVFtd4FrNKZJbKszxxDm+DPfB5j+hRC4oeEJ7w+sVyh3EawtfM +V7Mwj8i+7c3YUCravXBhSwG7SCTggFUgA8lMr8oWVgCATYsAEQEAAYkCPAQYAQoA +JhYhBMgEqT02vgxzPqGWRHwQYLf+UHATBQJe688BAhsMBQkSzAMAAAoJEHwQYLf+ +UHATTtwQAJiztPW68ykifpFdwYFp1VC7c+uGLhWBqjDY9NSUKNC9caR7bV0cnNu8 +07UG6j18gCB2GSkukXjOR/oTj6rNcW/WouPYfQOrw7+M2Ya8M8iq+E/HOXaXB3b4 +FeCcB0UuwfcHHd2KbXrRHA+9GNpmuOcfTCdsPpIr41Xg4QltATDEt/FrzuKspXg4 +vUKDXgfnbj7y0JcJM2FfcwWGlnAG5MMRyjJQAleGdiidX/9WxgJ4Mweq4qJM0jr3 +Qsrc9VuzxsLr85no3Hn5UYVgT7bBZ59HUbQoi775m78MxN3mWUSdcyLQKovI+YXr +tshTxWIf/2Ovdzt6Wq1WWXOGGuK1qgdPJTFWrlh3amFdb70zR1p6A/Lthd7Zty+n +QjRZRQo5jBSnYtjhMrZP6rxM3QqnQ0frEKK9HfDYONk1Bw18CUtdwFGb9OMregLR +IjvNLp9coSh5yYAepZyUGEPRET0GsmVw2trQF0uyMSkQfiq2zjPto6WWbsmrrbLr +cfZ/wnBw1FoNEd51U54euo9yvOgOVtJGvqLgHNwB8574FhQhoWAMhyizqdgeEt26 +m3FXecUNKL/AK71/l04vor+/WsXe8uhDg3O84qeYa9wgd8LZZVmGZJDosSwqYjtb +LdNNm+v60Zo6rFWSREegqi/nRTTDdxdW99ybjlh+mpbq3xavyFXF +=bhkm +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 676b908ce..326ad39da 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -162,6 +162,23 @@ get_random_value() { head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 } +gpg_rpm_import() { + if [ $OS = 'centos' ]; then + if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then + local RPMKEYSLOC=$temp_install_dir/salt/common/keys + else + local RPMKEYSLOC=$UPDATEDIR/salt/common/keys + fi + + RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') + + for RPMKEY in "${RPMKEYS[@]}"; do + rpm --import $RPMKEYSLOC/$RPMKEY + echo "Imported $RPMKEY" + done + fi +} + header() { printf '%s\n' "" "$banner" " $*" "$banner" } diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6ff298770..cb2d19aed 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -24,6 +24,7 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 SOUP_LOG=/root/soup.log +WHATWOULDYOUSAYYAHDOHERE=soup add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index a37d898af..1cc6bf6d9 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -20,6 +20,7 @@ Remap-npm: registry.npmjs.org Remap-node: nodejs.org Remap-apache: file:apache_mirrors ; file:backends_apache.us Remap-salt: repo.saltstack.com; https://repo.saltstack.com +Remap-securityonion: http://repocache.securityonion.net ; file:securityonion # Remap-secdeb: security.debian.org ReportPage: acng-report.html # SocketPath:/var/run/apt-cacher-ng/socket diff --git a/setup/so-functions b/setup/so-functions index b6cf569fb..d98a833da 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -159,11 +159,6 @@ check_network_manager_conf() { systemctl restart NetworkManager } >> "$setup_log" 2>&1 fi - - #if test -f "$nmconf"; then -# sed -i 's/managed=false/managed=true/g' "$nmconf" >> "$setup_log" 2>&1 -# systemctl restart NetworkManager >> "$setup_log" 2>&1 -# fi if [[ ! -d "$preupdir" ]]; then mkdir "$preupdir" >> "$setup_log" 2>&1 @@ -1054,40 +1049,11 @@ disable_ipv6() { } >> /etc/sysctl.conf } -#disable_misc_network_features() { -# filter_unused_nics -# if [ ${#filtered_nics[@]} -ne 0 ]; then -# for unused_nic in "${filtered_nics[@]}"; do -# if [ -n "$unused_nic" ]; then -# echo "Disabling unused NIC: $unused_nic" >> "$setup_log" 2>&1 -# -# # Disable DHCPv4/v6 and autoconnect -# nmcli con mod "$unused_nic" \ -# ipv4.method disabled \ -# ipv6.method ignore \ -# connection.autoconnect "no" >> "$setup_log" 2>&1 -# -# # Flush any existing IPs -# ip addr flush "$unused_nic" >> "$setup_log" 2>&1 -# fi -# done -# fi -# # Disable IPv6 -# { -# echo "net.ipv6.conf.all.disable_ipv6 = 1" -# echo "net.ipv6.conf.default.disable_ipv6 = 1" -# echo "net.ipv6.conf.lo.disable_ipv6 = 1" -# } >> /etc/sysctl.conf -#} - docker_install() { if [ $OS = 'centos' ]; then { yum clean expire-cache; - #if [[ ! $is_airgap ]]; then - # yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; - #fi if [[ ! $is_iso ]]; then yum -y install docker-ce-20.10.5-3.el7 containerd.io-1.4.4-3.1.el7; fi @@ -1988,11 +1954,6 @@ saltify() { # Install updates and Salt if [ $OS = 'centos' ]; then - set_progress_str 5 'Installing Salt repo' - { - sudo rpm --import https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub; - #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; - } >> "$setup_log" 2>&1 set_progress_str 6 'Installing various dependencies' if [[ ! $is_iso ]]; then logCmd "yum -y install wget nmap-ncat" @@ -2001,7 +1962,6 @@ saltify() { 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORT') reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then - #logCmd "yum -y install epel-release" logCmd "yum -y install sqlite argon2 curl mariadb-devel" fi # Download Ubuntu Keys in case manager updates = 1 @@ -2010,7 +1970,6 @@ saltify() { logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" logCmd "wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH" - #logCmd "cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo" fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then @@ -2035,13 +1994,11 @@ saltify() { ;; esac if [[ ! $is_airgap ]]; then - #cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo >> "$setup_log" 2>&1 yum clean expire-cache >> "$setup_log" 2>&1 fi set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - #yum -y install epel-release yum -y install salt-minion-3002.5\ python3\ python36-docker\ @@ -2266,10 +2223,14 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [ "$OS" = 'centos' ]; then + if [[ "$OS" == "centos" ]]; then mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ - cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ./yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + fi else echo "This is Ubuntu" fi diff --git a/setup/so-setup b/setup/so-setup index a532158f4..5989012ce 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -632,7 +632,9 @@ set_redirect >> $setup_log 2>&1 fi set_progress_str 2 'Updating packages' - update_packages >> $setup_log 2>&1 + if [[ ! $is_airgap ]] + update_packages >> $setup_log 2>&1 + fi if [[ $is_sensor || $is_helix || $is_import ]]; then set_progress_str 3 'Generating sensor pillar' diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo new file mode 100644 index 000000000..a38bac944 --- /dev/null +++ b/setup/yum_repos/securityonioncache.repo @@ -0,0 +1,56 @@ +[base] +name=CentOS-$releasever - Base +baseurl=https://repocache.securityonion.net/file/securityonion-repo/base/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#released updates +[updates] +name=CentOS-$releasever - Updates +baseurl=http://repocache.securityonion.net/file/securityonion-repo/updates/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that may be useful +[extras] +name=CentOS-$releasever - Extras +baseurl=http://repocache.securityonion.net/file/securityonion-repo/extras/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that extend functionality of existing packages +[centosplus] +name=CentOS-$releasever - Plus +baseurl=http://repocache.securityonion.net/file/securityonion-repo/centosplus/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +[epel] +name=Extra Packages for Enterprise Linux 7 - $basearch +baseurl=http://repocache.securityonion.net/file/securityonion-repo/epel/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/RPM-GPG-KEY-EPEL-7 + +[docker-ce-stable] +name=Docker CE Stable - $basearch +baseurl=http://repocache.securityonion.net/file/securityonion-repo/docker-ce-stable +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub + +[saltstack] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +enabled=1 +gpgcheck=1 +gpgkey=https://repocache.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + +[wazuh_repo] +gpgcheck=1 +gpgkey=http://repocache.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 \ No newline at end of file From 26f8ae87c59c56dc621e9cfd5a8c763ca0851494 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:10:00 -0400 Subject: [PATCH 025/258] Fix acng to actually cache --- setup/so-setup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 5989012ce..edbb5b408 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -47,6 +47,7 @@ source ./so-variables # Parse command line arguments setup_type=$1 automation=$2 +WHATWOULDYOUSAYYAHDOHERE=setup while [[ $# -gt 0 ]]; do arg="$1" @@ -622,8 +623,6 @@ set_redirect >> $setup_log 2>&1 info "Creating airgap repo" create_repo >> $setup_log 2>&1 airgap_rules >> $setup_log 2>&1 - else - securityonion_repo >> $setup_log 2>&1 fi if [[ $is_minion ]]; then @@ -632,7 +631,11 @@ set_redirect >> $setup_log 2>&1 fi set_progress_str 2 'Updating packages' + # Import the gpg keys + gpg_rpm_import if [[ ! $is_airgap ]] + securityonion_repo >> $setup_log 2>&1 + gpg_rpm_import >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 fi From 955d41abde6522c5834e2c93b188a39921ff8690 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:18:49 -0400 Subject: [PATCH 026/258] Fix acng to actually cache --- salt/common/tools/sbin/so-common | 2 +- setup/so-setup | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 326ad39da..985042876 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -163,7 +163,7 @@ get_random_value() { } gpg_rpm_import() { - if [ $OS = 'centos' ]; then + if [[ "$OS" = "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC=$temp_install_dir/salt/common/keys else diff --git a/setup/so-setup b/setup/so-setup index edbb5b408..7d42f94bc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -635,7 +635,6 @@ set_redirect >> $setup_log 2>&1 gpg_rpm_import if [[ ! $is_airgap ]] securityonion_repo >> $setup_log 2>&1 - gpg_rpm_import >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 fi From bab062e52b7bd8cef5cae153edf0fb22c6063648 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:21:03 -0400 Subject: [PATCH 027/258] Fix acng to actually cache --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 985042876..53901a73a 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -163,7 +163,7 @@ get_random_value() { } gpg_rpm_import() { - if [[ "$OS" = "centos" ]]; then + if [[ "$OS" == "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC=$temp_install_dir/salt/common/keys else From 362bf555260b4141bb6dd796da6fdf10d12df582 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 28 Mar 2021 22:01:58 -0400 Subject: [PATCH 028/258] fixpath for GPG keys --- setup/so-functions | 1 + setup/so-setup | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d98a833da..99819e9a9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2226,6 +2226,7 @@ securityonion_repo() { if [[ "$OS" == "centos" ]]; then mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ./yum_repos/securityonioncache.repo /etc/yum.repos.d/ else diff --git a/setup/so-setup b/setup/so-setup index 7d42f94bc..9beb11cec 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -632,8 +632,8 @@ set_redirect >> $setup_log 2>&1 set_progress_str 2 'Updating packages' # Import the gpg keys - gpg_rpm_import - if [[ ! $is_airgap ]] + gpg_rpm_import >> $setup_log 2>&1 + if [[ ! $is_airgap ]]; then securityonion_repo >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 fi From 5882642c3255b43714c8195d1947f9cbf1f993ff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 28 Mar 2021 22:10:02 -0400 Subject: [PATCH 029/258] fixpath for GPG Keys for real --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 53901a73a..04fcf529d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -165,9 +165,9 @@ get_random_value() { gpg_rpm_import() { if [[ "$OS" == "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then - local RPMKEYSLOC=$temp_install_dir/salt/common/keys + local RPMKEYSLOC="../salt/common/keys" else - local RPMKEYSLOC=$UPDATEDIR/salt/common/keys + local RPMKEYSLOC="$UPDATEDIR/salt/common/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') From d889bd26946775839a68ec20faed79d15533aaeb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 28 Mar 2021 22:32:03 -0400 Subject: [PATCH 030/258] Fix Security Onio Pub Key --- salt/common/keys/securityonion.pub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/keys/securityonion.pub b/salt/common/keys/securityonion.pub index c9148ff5c..15be14ca9 100644 --- a/salt/common/keys/securityonion.pub +++ b/salt/common/keys/securityonion.pub @@ -49,4 +49,4 @@ cfZ/wnBw1FoNEd51U54euo9yvOgOVtJGvqLgHNwB8574FhQhoWAMhyizqdgeEt26 m3FXecUNKL/AK71/l04vor+/WsXe8uhDg3O84qeYa9wgd8LZZVmGZJDosSwqYjtb LdNNm+v60Zo6rFWSREegqi/nRTTDdxdW99ybjlh+mpbq3xavyFXF =bhkm ------END PGP PUBLIC KEY BLOCK----- \ No newline at end of file +-----END PGP PUBLIC KEY BLOCK----- From 6bce8e8e2cffb4ee655eea8671767cde22c869b4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 29 Mar 2021 07:30:26 -0400 Subject: [PATCH 031/258] Remove incompatible example --- salt/soc/files/soc/custom.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/soc/files/soc/custom.js b/salt/soc/files/soc/custom.js index b23b7c36b..575e019a7 100644 --- a/salt/soc/files/soc/custom.js +++ b/salt/soc/files/soc/custom.js @@ -17,8 +17,5 @@ suggested to avoid and/or minimize the extent of any content placed here so that upgrading to newer version of Security Onion do not become a burden. - - Example: - - i18n.translations["en-US"].loginHeader = "Unauthorized use of this computer system is prohibited..."; + */ From 2ff790699fc9960c85d1566a3558f9ca5840e87c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 29 Mar 2021 09:36:24 -0400 Subject: [PATCH 032/258] [fix] Set ntp_string to empty, not ntp_servers --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 6dd10096b..533a77a92 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -489,7 +489,7 @@ collect_node_ls_pipeline_worker_count() { collect_ntp_servers() { if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then if whiptail_ntp_ask; then - [[ $is_airgap ]] && ntp_servers=() + [[ $is_airgap ]] && ntp_string="" whiptail_ntp_servers "$ntp_string" while ! valid_ntp_list "$ntp_string"; do From 1a58479f39178207ce08c7b028e5af7de2e7c9a8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Mar 2021 15:15:34 -0400 Subject: [PATCH 033/258] Fix acng passthrough --- salt/manager/files/acng/acng.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index 1cc6bf6d9..3492cf111 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -80,7 +80,7 @@ RedirMax: 6 VfileUseRangeOps: 0 # PassThroughPattern: private-ppa\.launchpad\.net:443$ # PassThroughPattern: .* # this would allow CONNECT to everything -PassThroughPattern: (download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ +PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ # ResponseFreezeDetectTime: 500 # ReuseConnections: 1 # PipelineDepth: 255 From f73bf947bcbd662d8693296b3642bdb2764722a6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Mar 2021 15:42:26 -0400 Subject: [PATCH 034/258] Fix repo url --- setup/yum_repos/securityonioncache.repo | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo index a38bac944..a55ee47d4 100644 --- a/setup/yum_repos/securityonioncache.repo +++ b/setup/yum_repos/securityonioncache.repo @@ -1,6 +1,6 @@ [base] name=CentOS-$releasever - Base -baseurl=https://repocache.securityonion.net/file/securityonion-repo/base/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/base/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 @@ -45,11 +45,11 @@ name=SaltStack repo for RHEL/CentOS $releasever PY3 baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ enabled=1 gpgcheck=1 -gpgkey=https://repocache.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub [wazuh_repo] gpgcheck=1 -gpgkey=http://repocache.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ From 3fce63e0c5e4a6142a5b998c29e03092c4119644 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Mar 2021 16:43:44 -0400 Subject: [PATCH 035/258] Fix Repo Again --- setup/yum_repos/securityonioncache.repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo index a55ee47d4..4fcb992d5 100644 --- a/setup/yum_repos/securityonioncache.repo +++ b/setup/yum_repos/securityonioncache.repo @@ -52,5 +52,5 @@ gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository -baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ protect=1 \ No newline at end of file From 0e9ffe033d86ce9ab78df1d6c1d157692ef6e40c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 09:30:06 -0400 Subject: [PATCH 036/258] Show message about setting up network earlier during setup --- setup/so-setup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d9e64105a..982195703 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -298,6 +298,10 @@ if ! [[ -f $install_opt_file ]]; then source "$net_init_file" fi + if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then + whiptail_management_interface_setup + fi + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init fi @@ -315,10 +319,6 @@ if ! [[ -f $install_opt_file ]]; then [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi - if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then - whiptail_management_interface_setup - fi - if [[ $is_minion ]]; then add_mngr_ip_to_hosts fi From 25eca39428a585557183535c3424b0cda00d9479 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 09:54:21 -0400 Subject: [PATCH 037/258] Always ask for ntp setup on iso installs, don't ask on network installs --- setup/so-functions | 48 ++++++++++++++++++---------------------------- setup/so-setup | 5 ++--- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 533a77a92..5c69b817a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -487,20 +487,18 @@ collect_node_ls_pipeline_worker_count() { } collect_ntp_servers() { - if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then - if whiptail_ntp_ask; then - [[ $is_airgap ]] && ntp_string="" + if whiptail_ntp_ask; then + [[ $is_airgap ]] && ntp_string="" + whiptail_ntp_servers "$ntp_string" + + while ! valid_ntp_list "$ntp_string"; do + whiptail_invalid_input whiptail_ntp_servers "$ntp_string" + done - while ! valid_ntp_list "$ntp_string"; do - whiptail_invalid_input - whiptail_ntp_servers "$ntp_string" - done - - IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array - else - ntp_servers=() - fi + IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array + else + ntp_servers=() fi } @@ -725,21 +723,19 @@ configure_ntp() { # Install chrony if it isn't already installed if ! command -v chronyc &> /dev/null; then - if [ "$OS" == centos ]; then - yum -y install chrony - else - retry 50 10 "apt-get -y install chrony" || exit 1 - fi + yum -y install chrony fi - [[ -f $chrony_conf ]] && rm -f $chrony_conf + [[ -f $chrony_conf ]] && mv $chrony_conf "$chrony_conf.bak" + + echo "# Config created by Security Onion" > $chrony_conf # Build list of servers for addr in "${ntp_servers[@]}"; do echo "server $addr iburst" >> $chrony_conf done - printf '%s\n' \ + printf '%s\n\n' \ 'driftfile /var/lib/chrony/drift' \ 'makestep 1.0 3' \ 'rtcsync' \ @@ -748,10 +744,10 @@ configure_ntp() { systemctl enable chronyd systemctl start chronyd - # Sync time & update the system time - chronyc -a 'burst 4/4' - sleep 20 # Wait for chrony to sync - chronyc -a makestep + # Tell the chrony daemon to sync time & update the system time + # Since these commands only make a call to chronyd, wait after each command to make sure the changes are made + chronyc -a 'burst 4/4' && sleep 30 + chronyc -a makestep && sleep 30 } checkin_at_boot() { @@ -761,12 +757,6 @@ checkin_at_boot() { echo "startup_states: highstate" >> "$minion_config" } -check_ntp_configured() { - if systemctl is-active --quiet chronyd || systemctl is-active --quiet ntpd; then - ntp_configured=true - fi -} - check_requirements() { local standalone_or_dist=$1 local node_type=$2 # optional diff --git a/setup/so-setup b/setup/so-setup index 982195703..37121c4fb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -534,8 +534,7 @@ if [[ $is_sensor && ! $is_eval ]]; then fi fi -check_ntp_configured -[[ -z $ntp_configured ]] && collect_ntp_servers +[[ $is_iso ]] && collect_ntp_servers if [[ $is_node && ! $is_eval ]]; then whiptail_node_advanced @@ -584,7 +583,7 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' - [[ -z $ntp_configured ]] && [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 + [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 reserve_ports From 5f5a53b8bb9dfb85e23f13fce913b4ea4fbe404b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Mar 2021 11:14:58 -0400 Subject: [PATCH 038/258] Push repolist to dev null --- salt/common/init.sls | 5 ----- setup/so-functions | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 157f2d49a..3e6774219 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -72,11 +72,6 @@ repair_yumdb: - onlyif: - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' -epel: - pkg.installed: - - skip_suggestions: True - - pkgs: - - epel-release {% endif %} # Install common packages diff --git a/setup/so-functions b/setup/so-functions index 99819e9a9..9adb22b86 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2644,6 +2644,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then + yum repolist >> /dev/null yum -y update >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 From 09064baf716afaad69837e64ce16d12434575253 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Mar 2021 11:21:19 -0400 Subject: [PATCH 039/258] Update so-common --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 04fcf529d..ccf211637 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -173,8 +173,8 @@ gpg_rpm_import() { RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') for RPMKEY in "${RPMKEYS[@]}"; do - rpm --import $RPMKEYSLOC/$RPMKEY - echo "Imported $RPMKEY" + rpm --import $RPMKEYSLOC/$RPMKEY + echo "Imported $RPMKEY" done fi } From fc3fd00216913b6446706a185b3dfdeafe95af1a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Mar 2021 11:28:47 -0400 Subject: [PATCH 040/258] Fix formatting --- setup/so-functions | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 9adb22b86..aff7a8375 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1978,19 +1978,6 @@ saltify() { systemctl enable salt-master >> "$setup_log" 2>&1 ;; *) - #if [ "$MANAGERUPDATES" = '1' ]; then - # { - # if [[ ! $is_airgap ]]; then - # Create the GPG Public Key for the Salt Repo - #cp ./public_keys/salt.pem /etc/pki/rpm-gpg/saltstack-signing-key; - - # Copy repo files over - #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; - # else - # info "This is airgap" - # fi - # } >> "$setup_log" 2>&1 - #fi ;; esac if [[ ! $is_airgap ]]; then @@ -2645,7 +2632,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then yum repolist >> /dev/null - yum -y update >> "$setup_log" + yum -y update >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 From 7049383ba68d162fe60f0f9391f391eeb1875012 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 30 Mar 2021 15:47:05 +0000 Subject: [PATCH 041/258] Add Elastic scripts --- .../tools/sbin/so-elasticsearch-indices-list | 21 ++++++++++++++++ .../tools/sbin/so-elasticsearch-pipeline-view | 25 +++++++++++++++++++ .../tools/sbin/so-elasticsearch-shards-list | 21 ++++++++++++++++ .../sbin/so-elasticsearch-template-remove | 21 ++++++++++++++++ .../tools/sbin/so-elasticsearch-template-view | 25 +++++++++++++++++++ .../tools/sbin/so-kibana-space-defaults | 0 salt/common/tools/sbin/so-logstash-events | 25 +++++++++++++++++++ .../tools/sbin/so-logstash-pipeline-stats | 25 +++++++++++++++++++ 8 files changed, 163 insertions(+) create mode 100755 salt/common/tools/sbin/so-elasticsearch-indices-list create mode 100755 salt/common/tools/sbin/so-elasticsearch-pipeline-view create mode 100755 salt/common/tools/sbin/so-elasticsearch-shards-list create mode 100755 salt/common/tools/sbin/so-elasticsearch-template-remove create mode 100755 salt/common/tools/sbin/so-elasticsearch-template-view mode change 100644 => 100755 salt/common/tools/sbin/so-kibana-space-defaults create mode 100755 salt/common/tools/sbin/so-logstash-events create mode 100755 salt/common/tools/sbin/so-logstash-pipeline-stats diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-list b/salt/common/tools/sbin/so-elasticsearch-indices-list new file mode 100755 index 000000000..c9df67a25 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-indices-list @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view new file mode 100755 index 000000000..04901e122 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . +else + curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . +fi diff --git a/salt/common/tools/sbin/so-elasticsearch-shards-list b/salt/common/tools/sbin/so-elasticsearch-shards-list new file mode 100755 index 000000000..9d28ed95b --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-shards-list @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +curl -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-template-remove b/salt/common/tools/sbin/so-elasticsearch-template-remove new file mode 100755 index 000000000..f7c3e6812 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-template-remove @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +curl -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 diff --git a/salt/common/tools/sbin/so-elasticsearch-template-view b/salt/common/tools/sbin/so-elasticsearch-template-view new file mode 100755 index 000000000..c9f3ec199 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-template-view @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . +else + curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . +fi diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/common/tools/sbin/so-kibana-space-defaults old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-logstash-events b/salt/common/tools/sbin/so-logstash-events new file mode 100755 index 000000000..817cafb72 --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-events @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done +else + curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events +fi diff --git a/salt/common/tools/sbin/so-logstash-pipeline-stats b/salt/common/tools/sbin/so-logstash-pipeline-stats new file mode 100755 index 000000000..b82a125d2 --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-pipeline-stats @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines +else + curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1 +fi From 679925ebd967160aa3242405127e3ce5b829c07b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 30 Mar 2021 13:29:56 -0400 Subject: [PATCH 042/258] Fix sensor cleanup & playbook sync scripts --- salt/common/tools/sbin/so-playbook-sync | 4 ++++ salt/common/tools/sbin/so-sensor-clean | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-playbook-sync b/salt/common/tools/sbin/so-playbook-sync index 250e4a3ad..a76d398cb 100755 --- a/salt/common/tools/sbin/so-playbook-sync +++ b/salt/common/tools/sbin/so-playbook-sync @@ -17,4 +17,8 @@ . /usr/sbin/so-common +# Check to see if we are already running +IS_RUNNING=$(ps aux | pgrep -f "so-playbook-sync" | wc -l) +[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0 + docker exec so-soctopus python3 playbook_play-sync.py diff --git a/salt/common/tools/sbin/so-sensor-clean b/salt/common/tools/sbin/so-sensor-clean index 63f102f0c..e62c3c4da 100755 --- a/salt/common/tools/sbin/so-sensor-clean +++ b/salt/common/tools/sbin/so-sensor-clean @@ -115,7 +115,7 @@ clean() { } # Check to see if we are already running -IS_RUNNING=$(ps aux | grep "so-sensor-clean" | grep -v grep | wc -l) +IS_RUNNING=$(ps aux | pgrep -f "so-sensor-clean" | wc -l) [ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then From be6eb3ed6c91495a79c8e760f0cf15372a5eee16 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 14:17:05 -0400 Subject: [PATCH 043/258] Restart chrony in case it's already running --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 5c69b817a..87c9b4885 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -742,7 +742,7 @@ configure_ntp() { 'logdir /var/log/chrony' >> $chrony_conf systemctl enable chronyd - systemctl start chronyd + systemctl restart chronyd # Tell the chrony daemon to sync time & update the system time # Since these commands only make a call to chronyd, wait after each command to make sure the changes are made From fd51b327ee1d860221c88dbb30c7c16a9cd105c5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 15:23:57 -0400 Subject: [PATCH 044/258] Add messaging to explain chronyc output to log --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 87c9b4885..d31eb28a3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -746,7 +746,9 @@ configure_ntp() { # Tell the chrony daemon to sync time & update the system time # Since these commands only make a call to chronyd, wait after each command to make sure the changes are made + printf "Syncing chrony time to server: " chronyc -a 'burst 4/4' && sleep 30 + printf "Forcing chrony to update the time: " chronyc -a makestep && sleep 30 } From 177989269fda48a1fa833cbf3e75e2379e07ed1b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 15:50:37 -0400 Subject: [PATCH 045/258] Better formatting of chrony.conf --- setup/so-functions | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d31eb28a3..862854c69 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -728,14 +728,16 @@ configure_ntp() { [[ -f $chrony_conf ]] && mv $chrony_conf "$chrony_conf.bak" - echo "# Config created by Security Onion" > $chrony_conf + printf '%s\n' "# NTP server list" > $chrony_conf # Build list of servers for addr in "${ntp_servers[@]}"; do echo "server $addr iburst" >> $chrony_conf done - printf '%s\n\n' \ + printf '\n%s\n' "# Config options" >> $chrony_conf + + printf '%s\n' \ 'driftfile /var/lib/chrony/drift' \ 'makestep 1.0 3' \ 'rtcsync' \ From 0b9cf57b5f10b73dfffd094e414fca5ebe75042f Mon Sep 17 00:00:00 2001 From: gebhard73 Date: Wed, 31 Mar 2021 14:22:06 +0200 Subject: [PATCH 046/258] Update so-index-list Sort by index name. --- salt/common/tools/sbin/so-index-list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list index dcfebbf58..cf9232150 100755 --- a/salt/common/tools/sbin/so-index-list +++ b/salt/common/tools/sbin/so-index-list @@ -15,4 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -curl -X GET -k -L https://localhost:9200/_cat/indices?v +curl -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" From 942de130caabc46726b7f658de6bf083a53cc60b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 31 Mar 2021 12:24:51 +0000 Subject: [PATCH 047/258] Enforce date type for ingest.timestamp --- salt/elasticsearch/templates/so/so-common-template.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index ebf123fed..c1f0a6755 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -267,9 +267,14 @@ }, "ingest":{ "type":"object", - "dynamic": true + "dynamic": true, + "properties":{ + "timestamp":{ + "type":"date" + } + } }, - "intel":{ + "intel":{ "type":"object", "dynamic": true, "properties":{ From 5203c25971d7bbcdab4fe5b76897ac60b73a0bb4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 09:13:38 -0400 Subject: [PATCH 048/258] Add Wazuh 4 Repo --- setup/yum_repos/securityonion.repo | 8 ++++++++ setup/yum_repos/securityonioncache.repo | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/setup/yum_repos/securityonion.repo b/setup/yum_repos/securityonion.repo index 20c907289..e61829380 100644 --- a/setup/yum_repos/securityonion.repo +++ b/setup/yum_repos/securityonion.repo @@ -53,4 +53,12 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 + +[wazuh4_repo] +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ protect=1 \ No newline at end of file diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo index 4fcb992d5..6d5058337 100644 --- a/setup/yum_repos/securityonioncache.repo +++ b/setup/yum_repos/securityonioncache.repo @@ -53,4 +53,12 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=http://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 + +[wazuh4_repo] +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ protect=1 \ No newline at end of file From c03e2b2c11279bb3fc7b1c53815b01207a252cfa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 09:14:40 -0400 Subject: [PATCH 049/258] Move ntp server array to its own pillar in the minion sls file --- setup/so-functions | 24 +++++++++++++++--------- setup/so-setup | 1 + 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 862854c69..702ccece3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1591,8 +1591,7 @@ manager_pillar() { printf '%s\n'\ " kratoskey: '$KRATOSKEY'"\ "" >> "$pillar_file" - - } +} manager_global() { local global_pillar="$local_salt_dir/pillar/global.sls" @@ -1620,12 +1619,7 @@ manager_global() { " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ - " managerip: '$MAINIP'"\ - " ntp_servers:" > "$global_pillar" - - for addr in "${ntp_servers[@]}"; do - echo " - '$addr'" >> "$global_pillar" - done + " managerip: '$MAINIP'" > "$global_pillar" if [[ $is_airgap ]]; then printf '%s\n'\ @@ -1774,7 +1768,6 @@ manager_global() { " bip: '$DOCKERBIP'"\ "redis_settings:"\ " redis_maxmemory: 812" >> "$global_pillar" - printf '%s\n' '----' >> "$setup_log" 2>&1 } @@ -1837,6 +1830,19 @@ network_setup() { } >> "$setup_log" 2>&1 } +ntp_pillar() { + local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls + + if [[ ${#ntp_servers[@]} -gt 0 ]]; then + printf '%s\n'\ + "ntp:"\ + " servers:" >> "$global_pillar" + for addr in "${ntp_servers[@]}"; do + printf '%s\n' " - '$addr'" >> "$pillar_file" + done + fi +} + parse_install_username() { # parse out the install username so things copy correctly INSTALLUSERNAME=${SUDO_USER:-${USER}} diff --git a/setup/so-setup b/setup/so-setup index 37121c4fb..65be15dc1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -584,6 +584,7 @@ set_redirect >> $setup_log 2>&1 set_progress_str 0 'Running initial configuration steps' [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 + ntp_pillar >> $setup_log 2>&1 reserve_ports From bb39ccc1aa512b72dc400b435030eecdb49356a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 09:25:21 -0400 Subject: [PATCH 050/258] Fix Automation Testing --- setup/automation/eval-net-centos | 2 +- setup/automation/import-net-centos | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/automation/eval-net-centos b/setup/automation/eval-net-centos index abd0c4765..82d2cc9ec 100644 --- a/setup/automation/eval-net-centos +++ b/setup/automation/eval-net-centos @@ -41,7 +41,7 @@ install_type=EVAL # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-net-centos b/setup/automation/import-net-centos index 37ca6ac51..f6394bde1 100644 --- a/setup/automation/import-net-centos +++ b/setup/automation/import-net-centos @@ -41,7 +41,7 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= From 46865809ed92cddb1199ab73a8bad9fd4c3de00c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 09:28:02 -0400 Subject: [PATCH 051/258] Fix Automation Testing round 2 --- setup/automation/import-airgap | 2 +- setup/automation/import-ami | 2 +- setup/automation/import-iso | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/automation/import-airgap b/setup/automation/import-airgap index bfd0e3641..9c394ef2f 100644 --- a/setup/automation/import-airgap +++ b/setup/automation/import-airgap @@ -42,7 +42,7 @@ INTERWEBS=AIRGAP # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-ami b/setup/automation/import-ami index 88734c352..10758be9a 100644 --- a/setup/automation/import-ami +++ b/setup/automation/import-ami @@ -41,7 +41,7 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-iso b/setup/automation/import-iso index 011623091..fbfdd364b 100644 --- a/setup/automation/import-iso +++ b/setup/automation/import-iso @@ -41,7 +41,7 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= From 89922a439e279a371973bab7b9ccd71bf2241cc7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 12:37:33 -0400 Subject: [PATCH 052/258] Move repo files --- salt/common/init.sls | 50 ++++++++++++++++++- .../common}/yum_repos/securityonion.repo | 0 .../common}/yum_repos/securityonioncache.repo | 0 setup/so-functions | 4 +- 4 files changed, 51 insertions(+), 3 deletions(-) rename {setup => salt/common}/yum_repos/securityonion.repo (100%) rename {setup => salt/common}/yum_repos/securityonioncache.repo (100%) diff --git a/salt/common/init.sls b/salt/common/init.sls index 3e6774219..d0dae49f5 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -64,7 +64,7 @@ salttmp: - group: 939 - makedirs: True -# Install epel +# Remove default Repos {% if grains['os'] == 'CentOS' %} repair_yumdb: cmd.run: @@ -72,6 +72,54 @@ repair_yumdb: - onlyif: - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' +crbase: + file.absent: + - name: /etc/yum.repos.d/CentOS-Base.repo + +crcr: + file.absent: + - name: /etc/yum.repos.d/CentOS-CR.repo + +crdebug: + file.absent: + - name: /etc/yum.repos.d/CentOS-Debuginfo.repo + +crfasttrack: + file.absent: + - name: /etc/yum.repos.d/CentOS-fasttrack.repo + +crmedia: + file.absent: + - name: /etc/yum.repos.d/CentOS-Media.repo + +crsources: + file.absent: + - name: /etc/yum.repos.d/CentOS-Sources.repo + +crvault: + file.absent: + - name: /etc/yum.repos.d/CentOS-Vault.repo + +crkernel: + file.absent: + - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo + +crepel: + file.absent: + - name: /etc/yum.repos.d/epel.repo + +crtesting: + file.absent: + - name: /etc/yum.repos.d/epel-testing.repo + +crssrepo: + file.absent: + - name: /etc/yum.repos.d/saltstack.repo + +crwazrepo: + file.absent: + - name: /etc/yum.repos.d/wazuh.repo + {% endif %} # Install common packages diff --git a/setup/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo similarity index 100% rename from setup/yum_repos/securityonion.repo rename to salt/common/yum_repos/securityonion.repo diff --git a/setup/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo similarity index 100% rename from setup/yum_repos/securityonioncache.repo rename to salt/common/yum_repos/securityonioncache.repo diff --git a/setup/so-functions b/setup/so-functions index aff7a8375..63cb5ca7c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2215,9 +2215,9 @@ securityonion_repo() { mv /etc/yum.repos.d/* /root/oldrepos/ rm -f /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ./yum_repos/securityonioncache.repo /etc/yum.repos.d/ + cp -f ../../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else - cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + cp -f ../../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ fi else echo "This is Ubuntu" From f8d72413544642bf9feab5abdc8e0a96c3e218e9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 12:55:46 -0400 Subject: [PATCH 053/258] Fix repo file path --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 63cb5ca7c..73293bc47 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2215,9 +2215,9 @@ securityonion_repo() { mv /etc/yum.repos.d/* /root/oldrepos/ rm -f /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else - cp -f ../../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ fi else echo "This is Ubuntu" From 1c4ba28336423c6164c671615e2d90d52e4fd4c4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 13:28:42 -0400 Subject: [PATCH 054/258] [fix] host_pillar overwrites the file, so run ntp_pillar after it --- setup/so-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 84e94e780..e2c866964 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -585,7 +585,6 @@ set_redirect >> $setup_log 2>&1 set_progress_str 0 'Running initial configuration steps' [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 - ntp_pillar >> $setup_log 2>&1 reserve_ports @@ -619,6 +618,8 @@ set_redirect >> $setup_log 2>&1 fi host_pillar >> $setup_log 2>&1 + ntp_pillar >> $setup_log 2>&1 + if [[ $is_minion || $is_import ]]; then set_updates >> $setup_log 2>&1 From 761a12ebbb6a95f8414f7b6e07b074a46ae025c2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 13:32:49 -0400 Subject: [PATCH 055/258] Fix variable name --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 6b4f693e3..2732a0ee9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1802,7 +1802,7 @@ ntp_pillar() { if [[ ${#ntp_servers[@]} -gt 0 ]]; then printf '%s\n'\ "ntp:"\ - " servers:" >> "$global_pillar" + " servers:" >> "$pillar_file" for addr in "${ntp_servers[@]}"; do printf '%s\n' " - '$addr'" >> "$pillar_file" done From 820b01405f428307aa23286ed7c9e9ef0c5f8a24 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 31 Mar 2021 14:57:36 -0400 Subject: [PATCH 056/258] For hunt quick actions, pipe value to 'escape' operator to escape backslashes and double quotes --- salt/soc/files/soc/alerts.actions.json | 2 +- salt/soc/files/soc/hunt.actions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/files/soc/alerts.actions.json b/salt/soc/files/soc/alerts.actions.json index 364c59d27..c0543d8fc 100644 --- a/salt/soc/files/soc/alerts.actions.json +++ b/salt/soc/files/soc/alerts.actions.json @@ -1,7 +1,7 @@ [ { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", "links": [ - "/#/hunt?q=\"{value}\" | groupby event.module event.dataset" + "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" ]}, { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", "links": [ diff --git a/salt/soc/files/soc/hunt.actions.json b/salt/soc/files/soc/hunt.actions.json index 364c59d27..c0543d8fc 100644 --- a/salt/soc/files/soc/hunt.actions.json +++ b/salt/soc/files/soc/hunt.actions.json @@ -1,7 +1,7 @@ [ { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", "links": [ - "/#/hunt?q=\"{value}\" | groupby event.module event.dataset" + "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" ]}, { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", "links": [ From f7e99b496134345e81ef8f787bb627997773358d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 31 Mar 2021 15:17:15 -0400 Subject: [PATCH 057/258] https://github.com/Security-Onion-Solutions/securityonion/issues/3709 --- salt/firewall/map.jinja | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index 2df668a07..496e6f568 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -18,14 +18,18 @@ {# This block translate the portgroups defined in the pillar to what is defined my portgroups.yaml and portgroups.local.yaml #} {% if salt['pillar.get']('firewall:assigned_hostgroups:chain') %} + {% set translated_pillar_assigned_hostgroups = {'chain': {}} %} {% for chain, hg in salt['pillar.get']('firewall:assigned_hostgroups:chain').items() %} {% for pillar_hostgroup, pillar_portgroups in salt['pillar.get']('firewall:assigned_hostgroups:chain')[chain].hostgroups.items() %} - {% do translated_pillar_assigned_hostgroups.update({"chain": {chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}}) %} + {% if translated_pillar_assigned_hostgroups.chain[chain] is defined %} + {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups.update({pillar_hostgroup: {"portgroups": []}}) %} + {% else %} + {% do translated_pillar_assigned_hostgroups.chain.update({chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}) %} + {% endif %} {% for pillar_portgroup in pillar_portgroups.portgroups %} {% set pillar_portgroup = pillar_portgroup.split('.') | last %} {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups[pillar_hostgroup].portgroups.append(defined_portgroups[pillar_portgroup]) %} - {% endfor %} {% endfor %} {% endfor %} @@ -39,7 +43,6 @@ {% set assigned_hostgroups = default_assigned_hostgroups.role[role] %} {% endif %} - {% if translated_pillar_assigned_hostgroups %} {% do salt['defaults.merge'](assigned_hostgroups, translated_pillar_assigned_hostgroups, merge_lists=True, in_place=True) %} {% endif %} \ No newline at end of file From ef984455605de8b1cb6f1ea7ee7b13d49afa824a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 31 Mar 2021 15:44:41 -0400 Subject: [PATCH 058/258] Fix Playbook Alert timestamps --- salt/elastalert/files/modules/so/playbook-es.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index c10a80f2c..ab2327ab7 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -17,7 +17,7 @@ class PlaybookESAlerter(Alerter): def alert(self, matches): for match in matches: today = strftime("%Y.%m.%d", gmtime()) - timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime()) + timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime()) headers = {"Content-Type": "application/json"} payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" From 3de980e4a1d0b9fd41aab9e87edf1293d5ba7fb6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 16:00:37 -0400 Subject: [PATCH 059/258] Move function call to run after Network Manager is installed --- setup/so-setup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index e2c866964..4a19b1b50 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -168,10 +168,8 @@ set_ssh_cmds $automated local_sbin="$(pwd)/../salt/common/tools/sbin" export PATH=$PATH:$local_sbin -set_network_dev_status_list set_palette >> $setup_log 2>&1 - # Kernel messages can overwrite whiptail screen #812 # https://github.com/Security-Onion-Solutions/securityonion/issues/812 dmesg -D @@ -434,6 +432,7 @@ if [[ $is_helix ]]; then fi if [[ $is_helix || $is_sensor ]]; then + set_network_dev_status_list whiptail_sensor_nics fi From 9b84a92ced276c4dc045e67cb501d89860c4a20f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:47:04 -0400 Subject: [PATCH 060/258] Manage the repo files --- salt/common/init.sls | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index d0dae49f5..1ee64bb5a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -2,6 +2,7 @@ {% if sls in allowed_states %} {% set role = grains.id.split('_') | last %} +{% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -84,6 +85,10 @@ crdebug: file.absent: - name: /etc/yum.repos.d/CentOS-Debuginfo.repo +crdockerce: + file.absent: + - name: /etc/yum.repos.d/docker-ce.repo + crfasttrack: file.absent: - name: /etc/yum.repos.d/CentOS-fasttrack.repo @@ -120,6 +125,17 @@ crwazrepo: file.absent: - name: /etc/yum.repos.d/wazuh.repo +crsecurityonionrepo: + file.managed: + {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] or managerupdates == 0 %} + - name: /etc/yum.repos.d/securityonion.repo + - source: salt://common/yum_repos.d/securityonion.repo + {% else %} + - name: /etc/yum.repos.d/securityonioncache.repo + - source: salt://commmon/yum_repos/securityonioncache.repo + {% endif %} + - mode: 644 + {% endif %} # Install common packages From 358f39753556883ba11034b3072dd970aa7a747f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:50:43 -0400 Subject: [PATCH 061/258] Manage the repo files --- salt/common/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 1ee64bb5a..c2ccb908f 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -88,7 +88,7 @@ crdebug: crdockerce: file.absent: - name: /etc/yum.repos.d/docker-ce.repo - + crfasttrack: file.absent: - name: /etc/yum.repos.d/CentOS-fasttrack.repo @@ -129,7 +129,7 @@ crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] or managerupdates == 0 %} - name: /etc/yum.repos.d/securityonion.repo - - source: salt://common/yum_repos.d/securityonion.repo + - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - source: salt://commmon/yum_repos/securityonioncache.repo From f387c4327a024f2f6993d5185cb5fe27628d51b3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:53:20 -0400 Subject: [PATCH 062/258] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index c2ccb908f..f15dfe70d 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -127,7 +127,7 @@ crwazrepo: crsecurityonionrepo: file.managed: - {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] or managerupdates == 0 %} + {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] %} - name: /etc/yum.repos.d/securityonion.repo - source: salt://common/yum_repos/securityonion.repo {% else %} From 13421bb04bb0b17f351c3f187382575c8c74a1dc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:59:15 -0400 Subject: [PATCH 063/258] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index f15dfe70d..0571cf2b3 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -132,7 +132,7 @@ crsecurityonionrepo: - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - - source: salt://commmon/yum_repos/securityonioncache.repo + - source: salt://common/yum_repos/securityonioncache.repo {% endif %} - mode: 644 From bfc5bb011f0d137a85dff07b64478cb8e3976219 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:03:52 -0400 Subject: [PATCH 064/258] Manage the repo files --- salt/common/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 0571cf2b3..5d95e69b6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -88,7 +88,7 @@ crdebug: crdockerce: file.absent: - name: /etc/yum.repos.d/docker-ce.repo - + crfasttrack: file.absent: - name: /etc/yum.repos.d/CentOS-fasttrack.repo @@ -127,9 +127,9 @@ crwazrepo: crsecurityonionrepo: file.managed: - {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] %} + {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - name: /etc/yum.repos.d/securityonion.repo - - source: salt://common/yum_repos/securityonion.repo + - source: salt://common/yum_repos.d/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - source: salt://common/yum_repos/securityonioncache.repo From 1509722185ebb4a8432edfcc0fe3dfc1fd27da4c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:04:56 -0400 Subject: [PATCH 065/258] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 5d95e69b6..f27af7233 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -128,7 +128,7 @@ crwazrepo: crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - - name: /etc/yum.repos.d/securityonion.repo + - name: /etc/yum.repos/securityonion.repo - source: salt://common/yum_repos.d/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo From 96459885556393912a373c49d72c9e2bc7714bf3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:06:26 -0400 Subject: [PATCH 066/258] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index f27af7233..2bfc6b66c 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -129,7 +129,7 @@ crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - name: /etc/yum.repos/securityonion.repo - - source: salt://common/yum_repos.d/securityonion.repo + - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - source: salt://common/yum_repos/securityonioncache.repo From 88eab865286fcd394a39b589868613c19f82dfc9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:07:30 -0400 Subject: [PATCH 067/258] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 2bfc6b66c..44a00024a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -128,7 +128,7 @@ crwazrepo: crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - - name: /etc/yum.repos/securityonion.repo + - name: /etc/yum.repos.d/securityonion.repo - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo From bc04cae91896c986575a44bcc0a812cb3d575d24 Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Thu, 1 Apr 2021 16:59:47 +0900 Subject: [PATCH 068/258] Fix: Connection to ES is "https" from 2.3.40 --- salt/curator/files/bin/so-curator-closed-delete-delete | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 58433ee1a..9cc94833c 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -34,7 +34,7 @@ overlimit() { closedindices() { - INDICES=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null) + INDICES=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null) [ $? -eq 1 ] && return false echo ${INDICES} | grep -q -E "(logstash-|so-)" } @@ -49,12 +49,12 @@ while overlimit && closedindices; do # First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed. # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. # Finally, select the first entry in that sorted list. - OLDEST_INDEX=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) + OLDEST_INDEX=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. - curl -XDELETE -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} + curl -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} # Finally, write a log entry that says we deleted it. echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG} -done \ No newline at end of file +done From 0d056123934fb754469191ac3ceb6e63abc04e40 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Apr 2021 10:00:55 -0400 Subject: [PATCH 069/258] Reserve ports for Zeek --- salt/common/files/99-reserved-ports.conf | 2 +- salt/common/init.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/files/99-reserved-ports.conf b/salt/common/files/99-reserved-ports.conf index a846341a5..208ef0acc 100644 --- a/salt/common/files/99-reserved-ports.conf +++ b/salt/common/files/99-reserved-ports.conf @@ -1 +1 @@ -net.ipv4.ip_local_reserved_ports=55000,57314 +net.ipv4.ip_local_reserved_ports=55000,57314,47760,47761,47762 diff --git a/salt/common/init.sls b/salt/common/init.sls index 3e6774219..6d0e567c5 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -268,7 +268,7 @@ docker: # Reserve OS ports for Docker proxy in case boot settings are not already applied/present dockerapplyports: cmd.run: - - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314"; fi + - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760,47761,47762"; fi # Reserve OS ports for Docker proxy dockerreserveports: From 40313fc2f5fd9b087f89bb1adda7f4ce0269da52 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Apr 2021 10:29:58 -0400 Subject: [PATCH 070/258] Reserve ports for Zeek --- salt/common/files/99-reserved-ports.conf | 2 +- salt/common/init.sls | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/files/99-reserved-ports.conf b/salt/common/files/99-reserved-ports.conf index 208ef0acc..ac4391693 100644 --- a/salt/common/files/99-reserved-ports.conf +++ b/salt/common/files/99-reserved-ports.conf @@ -1 +1 @@ -net.ipv4.ip_local_reserved_ports=55000,57314,47760,47761,47762 +net.ipv4.ip_local_reserved_ports=55000,57314,55000,57314,47760-47860 \ No newline at end of file diff --git a/salt/common/init.sls b/salt/common/init.sls index 6d0e567c5..7945a678a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -266,9 +266,10 @@ docker: - file: docker_daemon # Reserve OS ports for Docker proxy in case boot settings are not already applied/present +# 55000 = Wazuh, 57314 = Strelka, 47760-47860 = Zeek dockerapplyports: cmd.run: - - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760,47761,47762"; fi + - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760-47860"; fi # Reserve OS ports for Docker proxy dockerreserveports: From 7c6b037ae55ef36727e49b4b3786cc3fd8eff57c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Apr 2021 10:30:52 -0400 Subject: [PATCH 071/258] Reserve ports for Zeek --- salt/common/files/99-reserved-ports.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/files/99-reserved-ports.conf b/salt/common/files/99-reserved-ports.conf index ac4391693..82eb03f79 100644 --- a/salt/common/files/99-reserved-ports.conf +++ b/salt/common/files/99-reserved-ports.conf @@ -1 +1 @@ -net.ipv4.ip_local_reserved_ports=55000,57314,55000,57314,47760-47860 \ No newline at end of file +net.ipv4.ip_local_reserved_ports=55000,57314,47760-47860 \ No newline at end of file From 43c31b4e665fc993f6d5f6d07506ef19619cc924 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 1 Apr 2021 14:56:05 -0400 Subject: [PATCH 072/258] Fix script so changes are actually made --- salt/common/tools/sbin/so-ssh-harden | 70 ++++++++++++---------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 0e0b28b39..532e2f737 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -4,6 +4,8 @@ if [[ $1 =~ ^(-q|--quiet) ]]; then quiet=true +elif [[ $1 =~ ^(-v|--verbose) ]]; then + verbose=true fi sshd_config=/etc/ssh/sshd_config @@ -12,39 +14,27 @@ temp_config=/tmp/sshd_config before= after= reload_required=false +change_header_printed=false check_sshd_t() { local string=$1 - local state=$2 local grep_out grep_out=$(sshd -T | grep "^${string}") - if [[ $state == "Before" ]]; then - before=$grep_out - else - after=$grep_out - fi + before=$grep_out } print_diff() { - local type=$1 local diff - diff=$(diff -dqbB <(echo $before) <(echo $after)) + diff=$(diff -dbB <(echo $before) <(echo $after) | awk 'NR>1') if [[ -n $diff ]]; then - printf '%s\n' "$type" "$diff" - echo "" - fi -} - -print_msg() { - local msg=$1 - if ! [[ $quiet ]]; then - printf "%s\n" \ - "----" \ - "$msg" \ - "----" + if [[ $change_header_printed == false ]]; then + printf '%s\n' '' "Changes" '-------' '' + change_header_printed=true + fi + echo -e "$diff\n" fi } @@ -77,65 +67,65 @@ main() { echo "" >> $temp_config # Ciphers - check_sshd_t "ciphers" "Before" + check_sshd_t "ciphers" local cipher_string cipher_string=$(echo "$before" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g") - check_sshd_t "ciphers" "After" + after=$cipher_string - if ! [[ $quiet ]]; then print_diff "ciphers"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then + if [[ $before != "$after" ]]; then add_if_missing "$cipher_string" && test_config || exit 1 fi # KexAlgorithms - check_sshd_t "kexalgorithms" "Before" + check_sshd_t "kexalgorithms" local kexalg_string kexalg_string=$(echo "$before" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g") - check_sshd_t "kexalgorithms" "After" + after=$kexalg_string - if ! [[ $quiet ]]; then print_diff "kexalgorithms"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then + if [[ $before != "$after" ]]; then add_if_missing "$kexalg_string" && test_config || exit 1 fi # Macs - check_sshd_t "macs" "Before" + check_sshd_t "macs" local macs_string macs_string=$(echo "$before" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g") - check_sshd_t "macs" "After" + after=$macs_string - if ! [[ $quiet ]]; then print_diff "macs"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then - add_if_missing "$mac_string" && test_config || exit 1 + if [[ $before != "$after" ]]; then + add_if_missing "$macs_string" && test_config || exit 1 fi # HostKeyAlgorithms - check_sshd_t "hostkeyalgorithms" "Before" + check_sshd_t "hostkeyalgorithms" local hostkeyalg_string hostkeyalg_string=$(echo "$before" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g") - check_sshd_t "hostkeyalgorithms" "After" + after=$hostkeyalg_string - if ! [[ $quiet ]]; then print_diff "hostkeyalgorithms"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then + if [[ $before != "$after" ]]; then add_if_missing "$hostkeyalg_string" && test_config || exit 1 fi if [[ $reload_required == true ]]; then mv -f $temp_config $sshd_config - if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes..."; fi + if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes"; fi systemctl reload sshd - print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." + echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." else - if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up."; fi + if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up"; fi rm -f $temp_config fi } From fd57996bc6980ce7ddb7147f8bd43039337311e8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 2 Apr 2021 10:00:27 -0400 Subject: [PATCH 073/258] Change behavior of adding lines to sshd config * Replace existing lines in cases where a change has already been made --- salt/common/tools/sbin/so-ssh-harden | 20 +++++++++++--------- setup/so-setup | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 532e2f737..bdec9894b 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -38,12 +38,14 @@ print_diff() { fi } -add_if_missing() { - local string=$1 - if ! grep -q "$1" $temp_config; then - printf "%s\n\n" "$1" >> $temp_config - reload_required=true +replace_or_add() { + local type=$1 + local string=$2 + if grep -q "$type" $temp_config; then + sed -i "/$type .*/d" $temp_config fi + printf "%s\n\n" "$string" >> $temp_config + reload_required=true } test_config() { @@ -76,7 +78,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$cipher_string" && test_config || exit 1 + replace_or_add "ciphers" "$cipher_string" && test_config || exit 1 fi # KexAlgorithms @@ -90,7 +92,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$kexalg_string" && test_config || exit 1 + replace_or_add "kexalgorithms" "$kexalg_string" && test_config || exit 1 fi # Macs @@ -103,7 +105,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$macs_string" && test_config || exit 1 + replace_or_add "macs" "$macs_string" && test_config || exit 1 fi # HostKeyAlgorithms @@ -116,7 +118,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$hostkeyalg_string" && test_config || exit 1 + replace_or_add "hostkeyalgorithms" "$hostkeyalg_string" && test_config || exit 1 fi if [[ $reload_required == true ]]; then diff --git a/setup/so-setup b/setup/so-setup index a742630f7..584dc7933 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -906,7 +906,7 @@ set_redirect >> $setup_log 2>&1 set_progress_str 85 'Applying finishing touches' filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 - so-ssh-harden -q >> $setup_log 2>&1 + so-ssh-harden >> $setup_log 2>&1 if [[ $is_manager || $is_import ]]; then set_progress_str 87 'Adding user to SOC' From 8b8086b91a16b9e2a382d139f17db7651e9ddbdc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 2 Apr 2021 10:20:28 -0400 Subject: [PATCH 074/258] Update wording, as the new key tends to be ED25519, not ECDSA --- salt/common/tools/sbin/so-ssh-harden | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index bdec9894b..5e740c4dd 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -125,7 +125,7 @@ main() { mv -f $temp_config $sshd_config if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes"; fi systemctl reload sshd - echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." + echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the host key fingerprint for this server before reconnecting." else if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up"; fi rm -f $temp_config From d19c03efef69eaf4b9ddad6dea271764fad038bc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 2 Apr 2021 14:49:22 -0400 Subject: [PATCH 075/258] Refactor search of config lines * Create arrays for each line and loop through them for better code readability * Add more host key algorithms for removal * Update regex to look for a comma or EOL at the end of the search term, to avoid missing last item in list --- salt/common/tools/sbin/so-ssh-harden | 68 ++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 5e740c4dd..2a057ff5e 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -70,8 +70,23 @@ main() { # Ciphers check_sshd_t "ciphers" - local cipher_string - cipher_string=$(echo "$before" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g") + + local bad_ciphers=( + "3des-cbc" + "aes128-cbc" + "aes192-cbc" + "aes256-cbc" + "arcfour" + "arcfour128" + "arcfour256" + "blowfish-cbc" + "cast128-cbc" + ) + + local cipher_string=$before + for cipher in "${bad_ciphers[@]}"; do + cipher_string=$(echo "$cipher_string" | sed "s/${cipher}\(,\|\$\)//g" | sed 's/,$//') + done after=$cipher_string @@ -84,8 +99,20 @@ main() { # KexAlgorithms check_sshd_t "kexalgorithms" - local kexalg_string - kexalg_string=$(echo "$before" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g") + local bad_kexalgs=( + "diffie-hellman-group-exchange-sha1" + "diffie-hellman-group-exchange-sha256" + "diffie-hellman-group1-sha1" + "diffie-hellman-group14-sha1" + "ecdh-sha2-nistp256" + "ecdh-sha2-nistp521" + "ecdh-sha2-nistp384" + ) + + local kexalg_string=$before + for kexalg in "${bad_kexalgs[@]}"; do + kexalg_string=$(echo "$kexalg_string" | sed "s/${kexalg}\(,\|\$\)//g" | sed 's/,$//') + done after=$kexalg_string @@ -97,8 +124,21 @@ main() { # Macs check_sshd_t "macs" - local macs_string - macs_string=$(echo "$before" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g") + + local bad_macs=( + "hmac-sha2-512" + "umac-128@openssh.com" + "hmac-sha2-256" + "umac-64@openssh.com" + "hmac-sha1" + "hmac-sha1-etm@openssh.com" + "umac-64-etm@openssh.com" + ) + + local macs_string=$before + for mac in "${bad_macs[@]}"; do + macs_string=$(echo "$macs_string" | sed "s/${mac}\(,\|\$\)//g" | sed 's/,$//') + done after=$macs_string @@ -110,8 +150,20 @@ main() { # HostKeyAlgorithms check_sshd_t "hostkeyalgorithms" - local hostkeyalg_string - hostkeyalg_string=$(echo "$before" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g") + + local optional_suffix_regex_hka="\(-cert-v01@openssh.com\)\?" + local bad_hostkeyalg_list=( + "ecdsa-sha2-nistp256" + "ecdsa-sha2-nistp384" + "ecdsa-sha2-nistp521" + "ssh-rsa" + "ssh-dss" + ) + + local hostkeyalg_string=$before + for alg in "${bad_hostkeyalg_list[@]}"; do + hostkeyalg_string=$(echo "$hostkeyalg_string" | sed "s/${alg}${optional_suffix_regex_hka}\(,\|\$\)//g" | sed 's/,$//') + done after=$hostkeyalg_string From e430be1017b5a46e11b43bbb6bc97ebb7f398f67 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 2 Apr 2021 16:36:29 -0400 Subject: [PATCH 076/258] Enable Flux compatibility mode to prepare for eventual migration to 2.0 --- salt/influxdb/etc/influxdb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/etc/influxdb.conf b/salt/influxdb/etc/influxdb.conf index 86c1ccfe8..9d89ca774 100644 --- a/salt/influxdb/etc/influxdb.conf +++ b/salt/influxdb/etc/influxdb.conf @@ -233,7 +233,7 @@ # enabled = true # Determines whether the Flux query endpoint is enabled. - # flux-enabled = false + flux-enabled = true # The bind address used by the HTTP service. # bind-address = ":8086" From 8ca0626387b6c8f61a4e79b92180c2e816cc8f2d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 5 Apr 2021 06:55:40 -0400 Subject: [PATCH 077/258] FIX: Hunt query for HTTP EXE downloads should work for both Zeek and Suricata #3753 --- salt/soc/files/soc/hunt.queries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index 840b4b373..93295364d 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -34,7 +34,7 @@ { "name": "HTTP", "description": "HTTP grouped by status code and message", "query": "event.dataset:http | groupby http.status_code http.status_message"}, { "name": "HTTP", "description": "HTTP grouped by method and user agent", "query": "event.dataset:http | groupby http.method http.useragent"}, { "name": "HTTP", "description": "HTTP grouped by virtual host", "query": "event.dataset:http | groupby http.virtual_host"}, - { "name": "HTTP", "description": "HTTP with exe downloads", "query": "event.dataset:http AND file.resp_mime_types:dosexec | groupby http.virtual_host"}, + { "name": "HTTP", "description": "HTTP with exe downloads", "query": "event.dataset:http AND (file.resp_mime_types:dosexec OR file.resp_mime_types:executable) | groupby http.virtual_host"}, { "name": "Intel", "description": "Intel framework hits grouped by indicator", "query": "event.dataset:intel | groupby intel.indicator.keyword"}, { "name": "IRC", "description": "IRC grouped by command", "query": "event.dataset:irc | groupby irc.command.type"}, { "name": "KERBEROS", "description": "KERBEROS grouped by service", "query": "event.dataset:kerberos | groupby kerberos.service"}, From 534dbf9761ecd7680dfd0ebbe9eb8b77475edccc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 09:07:00 -0400 Subject: [PATCH 078/258] change the upgrade command - https://github.com/Security-Onion-Solutions/securityonion/issues/3501 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 7ef63bd68..80646f4ca 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -19,12 +19,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum clean all ; yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" ; /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common ; apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; apt-mark hold salt-common && apt-mark hold salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %} From 1c3a7094bdc8607c74315ace5a8a7e70c83b99f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 11:05:48 -0400 Subject: [PATCH 079/258] upgrade salt to 3003.1 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- setup/so-functions | 18 +++++++++--------- setup/so-preflight | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 2b50b517b..e97115ec1 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3002.5 \ No newline at end of file + version: 3003.1 \ No newline at end of file diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index e6b1303ed..3993c827b 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,5 +2,5 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3002.5 + version: 3003.1 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 2732a0ee9..75e8951ad 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2029,13 +2029,13 @@ saltify() { # Download Ubuntu Keys in case manager updates = 1 mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 if [[ ! $is_airgap ]]; then - logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" + logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" logCmd "wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH" fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then - logCmd "yum -y install salt-master-3002.5" + logCmd "yum -y install salt-master-3003.1" fi systemctl enable salt-master >> "$setup_log" 2>&1 ;; @@ -2048,7 +2048,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - yum -y install salt-minion-3002.5\ + yum -y install salt-minion-3003.1\ python3\ python36-docker\ python36-dateutil\ @@ -2100,8 +2100,8 @@ saltify() { 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR') # Add saltstack repo(s) - wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 - echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" + wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 + echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" # Add Docker repo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - >> "$setup_log" 2>&1 @@ -2109,7 +2109,7 @@ saltify() { # Get gpg keys mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 - wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com$py_ver_url_path/ubuntu/"$ubuntu_version"/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 + wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com$py_ver_url_path/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg >> "$setup_log" 2>&1 wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH >> "$setup_log" 2>&1 @@ -2122,7 +2122,7 @@ saltify() { set_progress_str 6 'Installing various dependencies' retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1 set_progress_str 7 'Installing salt-master' - retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-master=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 ;; *) @@ -2133,14 +2133,14 @@ saltify() { echo "Using apt-key add to add SALTSTACK-GPG-KEY.pub and GPG-KEY-WAZUH" >> "$setup_log" 2>&1 apt-key add "$temp_install_dir"/gpg/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 apt-key add "$temp_install_dir"/gpg/GPG-KEY-WAZUH >> "$setup_log" 2>&1 - echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" + echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" ;; esac retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 set_progress_str 8 'Installing salt-minion & python modules' - retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-minion=3003.1+ds-1 salt-common=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1 diff --git a/setup/so-preflight b/setup/so-preflight index da25e6775..1ef840284 100644 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -46,8 +46,8 @@ check_new_repos() { if [[ $OS == 'centos' ]]; then local repo_arr=( "https://download.docker.com/linux/centos/docker-ce.repo" - "https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/SALTSTACK-GPG-KEY.pub" - "https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3003/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" "https://download.docker.com/linux/ubuntu/gpg" "https://packages.wazuh.com/key/GPG-KEY-WAZUH" "https://packages.wazuh.com/3.x/yum/" @@ -59,7 +59,7 @@ check_new_repos() { local repo_arr=( "https://download.docker.com/linux/ubuntu/gpg" "https://download.docker.com/linux/ubuntu" - "https://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" "https://packages.wazuh.com/key/GPG-KEY-WAZUH" "https://packages.wazuh.com" ) From f9dc040c7fc35fded3986e7c7e0947ae365411cb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 11:38:39 -0400 Subject: [PATCH 080/258] Fix Raid --- salt/common/tools/sbin/so-raid-status | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index d55d158fe..11909e012 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -66,11 +66,13 @@ mkdir -p /opt/so/log/raid {%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %} #check_boss_raid check_software_raid -echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log +#echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log +echo "osraid=1 nsmraid=$SWRAID" > /opt/so/log/raid/status.log {%- elif grains['sosmodel'] in ['SOS1000F', 'SOS1000', 'SOSSN7200', 'SOS10K', 'SOS4000'] %} #check_boss_raid check_lsi_raid -echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log +#echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log +echo "osraid=1 nsmraid=$LSIRAID" > /opt/so/log/raid/status.log {%- else %} exit 0 {%- endif %} From 5aefa2a02408f7f128831e34776d6430f22e1f80 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 11:41:19 -0400 Subject: [PATCH 081/258] Fix Raid for Jertel compliance --- salt/telegraf/scripts/raid.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index c53644889..0938bb658 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -27,7 +27,7 @@ RAIDLOG=/var/log/raid/status.log RAIDSTATUS=$(cat /var/log/raid/status.log) if [ -f "$RAIDLOG" ]; then - echo "raid raidstatus=$RAIDSTATUS " + echo "raid $RAIDSTATUS" else exit 0 fi From bad22ab541ba746bcf4b15a1f727d98d6ee63758 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 12:08:38 -0400 Subject: [PATCH 082/258] Add model to sensoroni config --- salt/sensoroni/files/sensoroni.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 23b967b04..dead05b64 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,5 +1,9 @@ {%- set URLBASE = salt['pillar.get']('global:url_base') %} -{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %} +{%- if salt['pillar.get']('sensoroni:node_description') %} +{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %} +{%- else %} +{%- set DESCRIPTION = salt['grains.get']('sosmodel', '') %} +{%- endif %} {%- set ADDRESS = salt['pillar.get']('sensoroni:node_address') %} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} {%- set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) %} From a824813cdb95b3a6b86f57026d9729b0e807bb00 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 12:10:29 -0400 Subject: [PATCH 083/258] Add model to sensoroni config --- salt/sensoroni/files/sensoroni.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index dead05b64..df2990404 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -13,7 +13,7 @@ {%- else %} {%- set STENODEFAULT = False %} {%- endif %} -{%- set STENOENABLED = salt['pillar.get']('steno:enabled', STENODEFAULT) %} +{%- set STENOENABLED = salt['pillar.get']('steno:enabled', STENODEFAULT) %} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", From 9b8b5e6173b8b3b105dd570edc6c956c0bf08113 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 14:12:24 -0400 Subject: [PATCH 084/258] use -r by default to disable salt bootstrap from doing repo things --- salt/salt/map.jinja | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 80646f4ca..6b5273b84 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -18,11 +18,7 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} - {% if ISAIRGAP is sameas true %} {% set UPGRADECOMMAND = 'yum clean all ; yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} - {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" ; /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} - {% endif %} {% elif grains.os|lower == 'ubuntu' %} {% set UPGRADECOMMAND = 'apt-mark unhold salt-common ; apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; apt-mark hold salt-common && apt-mark hold salt-minion' %} {% endif %} From fc886341591d6d34cbdf1d4057f63aada1363ceb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 15:01:21 -0400 Subject: [PATCH 085/258] Set the Repo for airgap during install --- setup/so-functions | 11 +++++++++++ setup/so-setup | 2 ++ 2 files changed, 13 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index c9f0925ec..47d75f1c0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -44,6 +44,17 @@ logCmd() { } ### End Logging Section ### +airgap_repo() { + # Remove all the repo files + rm -rf /etc/yum.repos.d/* + echo "[airgap_repo]" > /etc/yum./repos.d/airgap_repo.repo + echo "baseurl=https://$MSRV/repo" >> /etc/yum.repos.d/airgap_repo.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/airgap_repo.repo + echo "sslverify=0" >> /etc/yum.repos.d/airgap_repo.repo + echo "name=Airgap Repo" >> /etc/yum.repos.d/airgap_repo.repo + echo "enabled=1" >> /etc/yum.repos.d/airgap_repo.repo +} + airgap_rules() { # Copy the rules for suricata if using Airgap mkdir -p /nsm/repo/rules diff --git a/setup/so-setup b/setup/so-setup index 0aa78aa10..d3ef48ba6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -648,6 +648,8 @@ set_redirect >> $setup_log 2>&1 if [[ ! $is_airgap ]]; then securityonion_repo >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 + else + airgap_repo >> $setup_log 2>&1 fi if [[ $is_sensor || $is_helix || $is_import ]]; then From 83bf709290e14a845e68fc90234374356c22acdc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 15:12:53 -0400 Subject: [PATCH 086/258] use -r for salt boostrap in soup as well --- salt/common/tools/sbin/soup | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index cb2d19aed..d06002b7d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,11 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - if [ $is_airgap -eq 0 ]; then - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" - else - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - fi + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 89f72bb6edfc8437c3681e95b6c7b91e62c2da70 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 16:44:51 -0400 Subject: [PATCH 087/258] check if . in new version, append .1 if not --- salt/common/tools/sbin/soup | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d06002b7d..eda93fc1f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -649,6 +649,12 @@ fi echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded +SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'}) +if [[ "." =~ .*"$SALTVERSIONPOSTUPGRADE".* ]]; then + SALTVERSIONPOSTUPGRADE=$SALTVERSIONPOSTUPGRADE +else + SALTVERSIONPOSTUPGRADE="${SALTVERSIONPOSTUPGRADE}.1" +fi if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." From 5f6770925d0ede70182da4c510f244d3793e3b14 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Apr 2021 16:52:12 -0400 Subject: [PATCH 088/258] speculative commit --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 018bdfac7..e889c64d0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1008,7 +1008,7 @@ create_repo() { detect_cloud() { echo "Testing if setup is running on a cloud instance..." | tee -a "$setup_log" - if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null); then export is_cloud="true"; fi + if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null) || [ -f /var/log/waagent.log ]; then export is_cloud="true"; fi } detect_os() { From 3a1c478d9a3e9f0415eeae9f5e8fe5fbb1c94f11 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 16:56:34 -0400 Subject: [PATCH 089/258] compare the new var --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index eda93fc1f..9036d83bb 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -655,7 +655,7 @@ if [[ "." =~ .*"$SALTVERSIONPOSTUPGRADE".* ]]; then else SALTVERSIONPOSTUPGRADE="${SALTVERSIONPOSTUPGRADE}.1" fi -if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) != "$NEWSALTVERSION" ]]; then +if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." echo "Exiting." From a38015bd989e0101ecafa3eb9c4285bf94c35750 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 17:28:04 -0400 Subject: [PATCH 090/258] Add some manager logic --- salt/common/init.sls | 3 +++ setup/so-functions | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 5fe3d9081..0ada77e1a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -3,6 +3,7 @@ {% set role = grains.id.split('_') | last %} {% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} +{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -125,6 +126,7 @@ crwazrepo: file.absent: - name: /etc/yum.repos.d/wazuh.repo +{% if not ISAIRGAP %} crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} @@ -137,6 +139,7 @@ crsecurityonionrepo: - mode: 644 {% endif %} +{% endif %} # Install common packages {% if grains['os'] != 'CentOS' %} diff --git a/setup/so-functions b/setup/so-functions index 47d75f1c0..55cd9290d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -48,7 +48,11 @@ airgap_repo() { # Remove all the repo files rm -rf /etc/yum.repos.d/* echo "[airgap_repo]" > /etc/yum./repos.d/airgap_repo.repo - echo "baseurl=https://$MSRV/repo" >> /etc/yum.repos.d/airgap_repo.repo + if $is_manager; then + echo "baseurl=https://$HOSTNAME/repo" >> /etc/yum.repos.d/airgap_repo.repo + else + echo "baseurl=https://$MSRV/repo" >> /etc/yum.repos.d/airgap_repo.repo + fi echo "gpgcheck=1" >> /etc/yum.repos.d/airgap_repo.repo echo "sslverify=0" >> /etc/yum.repos.d/airgap_repo.repo echo "name=Airgap Repo" >> /etc/yum.repos.d/airgap_repo.repo From 168d0bcaf4c1bbee023868651d7cebd9a36cc0fd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 18:30:07 -0400 Subject: [PATCH 091/258] Fix Spelling issue --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 55cd9290d..2e5e9ee7c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -47,7 +47,7 @@ logCmd() { airgap_repo() { # Remove all the repo files rm -rf /etc/yum.repos.d/* - echo "[airgap_repo]" > /etc/yum./repos.d/airgap_repo.repo + echo "[airgap_repo]" > /etc/yum.repos.d/airgap_repo.repo if $is_manager; then echo "baseurl=https://$HOSTNAME/repo" >> /etc/yum.repos.d/airgap_repo.repo else From 5525b9e97d17c472b58a23b42b6aecb8b2732d66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 08:30:57 -0400 Subject: [PATCH 092/258] point to new salt repo --- salt/common/yum_repos/securityonion.repo | 2 +- salt/common/yum_repos/securityonioncache.repo | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index e61829380..2fb35e579 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -42,7 +42,7 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..56ada1413 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -42,7 +42,7 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From 1ea0be00976a8f298b5e7c43b8c5cd54f270847c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 09:15:22 -0400 Subject: [PATCH 093/258] remove references to 3003.1 change to 3003 --- salt/common/tools/sbin/soup | 5 ----- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- salt/top.sls | 1 + setup/so-functions | 8 ++++---- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9036d83bb..0a68fae2d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -650,11 +650,6 @@ echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'}) -if [[ "." =~ .*"$SALTVERSIONPOSTUPGRADE".* ]]; then - SALTVERSIONPOSTUPGRADE=$SALTVERSIONPOSTUPGRADE -else - SALTVERSIONPOSTUPGRADE="${SALTVERSIONPOSTUPGRADE}.1" -fi if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index e97115ec1..8588af65c 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3003.1 \ No newline at end of file + version: 3003 \ No newline at end of file diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 3993c827b..9d888f106 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,5 +2,5 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3003.1 + version: 3003 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 68c392c25..6b522d03b 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -17,6 +17,7 @@ {% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {% import_yaml 'salt/minion.defaults.yaml' as saltversion %} {% set saltversion = saltversion.salt.minion.version %} +{% set INSTALLEDSALTVERSION = grains.saltversion %} base: diff --git a/setup/so-functions b/setup/so-functions index 8dd5d2f75..bc2908a41 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2033,7 +2033,7 @@ saltify() { fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then - logCmd "yum -y install salt-master-3003.1" + logCmd "yum -y install salt-master-3003" fi systemctl enable salt-master >> "$setup_log" 2>&1 ;; @@ -2046,7 +2046,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - yum -y install salt-minion-3003.1\ + yum -y install salt-minion-3003\ python3\ python36-docker\ python36-dateutil\ @@ -2120,7 +2120,7 @@ saltify() { set_progress_str 6 'Installing various dependencies' retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1 set_progress_str 7 'Installing salt-master' - retry 50 10 "apt-get -y install salt-master=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-master=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 ;; *) @@ -2138,7 +2138,7 @@ saltify() { retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 set_progress_str 8 'Installing salt-minion & python modules' - retry 50 10 "apt-get -y install salt-minion=3003.1+ds-1 salt-common=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1 From 049001d572fd0ad5f5801c32a448782dd9d04ea0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 09:48:21 -0400 Subject: [PATCH 094/258] set repo url for salt upgrade for centos --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0a68fae2d..d8007012e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 01f95c846c6a9403f4b704646e6effaf8b3156c1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 11:41:06 -0400 Subject: [PATCH 095/258] remove trailing / --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d8007012e..f36cdac68 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 521dbbd90a1f7441c39deb4191013dbb0e569005 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 11:45:59 -0400 Subject: [PATCH 096/258] change repo path --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f36cdac68..e14c9a0aa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003/ -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 6da84c7c87fb2b1b2519b06100fa026d2bbbbbf5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 12:00:36 -0400 Subject: [PATCH 097/258] strip trailing / --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e14c9a0aa..74a1c26c0 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003/ -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003 -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 80509fbbc61987eecf7b0bccd17d7c59b454a1e4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 12:23:11 -0400 Subject: [PATCH 098/258] fix -R repo option --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 74a1c26c0..5108e73d3 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003 -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From b70d9c089275206daa36f35b0837a07f534a19a5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 6 Apr 2021 13:20:46 -0400 Subject: [PATCH 099/258] Add end summary and warning about SSH host key change --- setup/so-setup | 9 +- setup/so-whiptail | 203 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 48322f246..509ad419d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -572,14 +572,14 @@ fi if [[ $is_manager || $is_import ]]; then collect_so_allow; fi -whiptail_make_changes +# This block sets REDIRECTIT which is used by a function outside the below subshell +set_redirect >> $setup_log 2>&1 + +whiptail_end_settings # From here on changes will be made. echo "1" > /root/accept_changes -# This block sets REDIRECTIT which is used by a function outside the below subshell -set_redirect >> $setup_log 2>&1 - # Begin install { @@ -962,6 +962,7 @@ else } | whiptail_gauge_post_setup "Running post-installation steps..." whiptail_setup_complete + [[ $setup_type != 'iso' ]] && whitpail_ssh_warning echo "Post-installation steps have completed." >> $setup_log 2>&1 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index c71db9508..45b263f96 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -423,6 +423,193 @@ whiptail_enable_components() { done } +whiptail_end_settings() { + [ -n "$TESTING" ] && return + + # BASIC INFO (NETWORK, HOSTNAME, DESCRIPTION, ETC) + + read -r -d '' end_msg <<- EOM + The following options have been set, would you like to proceed? + + Node Type: $install_type + Hostname: $HOSTNAME + EOM + + [[ -n $NODE_DESCRIPTION ]] && __append_end_msg "Description: $NODE_DESCRIPTION" + + [[ $is_airgap ]] && __append_end_msg "Airgap: True" + + if [[ $is_minion ]]; then + __append_end_msg "Manager Hostname: $MSRV" + __append_end_msg "Manager IP: $MSRVIP" + fi + + + [[ $is_iso ]] && __append_end_msg "Network: $address_type" + + __append_end_msg "Management NIC: $MNIC" + __append_end_msg "Management IP: $MAINIP" + + if [[ $address_type == 'STATIC' ]]; then + __append_end_msg "Gateway: $MGATEWAY" + __append_end_msg "DNS: $MDNS" + __append_end_msg "DNS Domain: $MSEARCH" + fi + + if [[ $is_sensor ]]; then + __append_end_msg "Bond NIC(s):" + for nic in "${BNICS[@]}"; do + __append_end_msg " - $nic" + done + fi + + local homenet_arr + if [[ -n $HNMANAGER ]]; then + __append_end_msg "Home Network(s):" + IFS="," read -r -a homenet_arr <<< "$HNMANAGER" + for net in "${homenet_arr[@]}"; do + __append_end_msg " - $net" + done + elif [[ -n $HNSENSOR ]]; then + __append_end_msg "Home Network(s):" + IFS="," read -r -a homenet_arr <<< "$HNSENSOR" + for net in "${homenet_arr[@]}"; do + __append_end_msg " - $net" + done + fi + + [[ -n $REDIRECTIT ]] && __append_end_msg "Access URL: https://${REDIRECTIT}" + + [[ -n $ALLOW_CIDR ]] && __append_end_msg "Allowed IP or Subnet: $ALLOW_CIDR" + + [[ -n $WEBUSER ]] && __append_end_msg "Web User: $WEBUSER" + + [[ -n $FLEETNODEUSER ]] && __append_end_msg "Fleet User: $FLEETNODEUSER" + + if [[ $is_manager ]]; then + __append_end_msg "Enabled Optional Components:" + for component in "${COMPONENTS[@]}"; do + __append_end_msg " - $component" + done + fi + + # METADATA / IDS + + if [[ -n $ZEEKVERSION ]]; then + local md_tool_string=${ZEEKVERSION,;} + md_tool_string=${md_tool_string^} + + __append_end_msg "Metadata Tool: $md_tool_string" + fi + + [[ -n $RULESETUP ]] && __append_end_msg "IDS Ruleset: $RULESETUP" + [[ -n $OINKCODE ]] && __append_end_msg "Oinkcode: $OINKCODE" + + # PATCH SCHEDULE + + [[ -n $PATCHSCHEDULENAME ]] && __append_end_msg "Patch schedule: $PATCHSCHEDULENAME" + + if [[ ${#PATCHSCHEDULEDAYS[@]} -gt 0 ]]; then + __append_end_msg "Day(s):" + for day in "${PATCHSCHEDULEDAYS[@]}"; do + __append_end_msg " - $day" + done + fi + + if [[ ${#PATCHSCHEDULEHOURS[@]} -gt 0 ]]; then + __append_end_msg "Hours(s):" + for hour in "${PATCHSCHEDULEHOURS[@]}"; do + __append_end_msg " - $hour" + done + fi + + # MISC + + [[ $is_helix ]] && __append_end_msg "Helix API key: $HELIXAPIKEY" + [[ -n $DOCKERNET ]] && __append_end_msg "Docker network: $DOCKERNET" + if [[ -n $MANAGERUPDATES ]]; then + __append_end_msg "OS Package Updates: Manager" + else + __append_end_msg "OS Package Updates: Open" + fi + if [[ ${#ntp_servers[@]} -gt 0 ]]; then + __append_end_msg "NTP Servers:" + for server in "${ntp_servers[@]}"; do + __append_end_msg " - $server" + done + fi + + # ADVANCED OR REGULAR + + if [[ $NODESETUP == 'NODEADVANCED' ]]; then + __append_end_msg "Advanced Node Settings:" + __append_end_msg " Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE" + __append_end_msg " Logstash Heap Size: $NODE_LS_HEAP_SIZE" + __append_end_msg " Logstash Worker Count: $LSPIPELINEWORKERS" + __append_end_msg " Logstash Batch Size: $LSPIPELINEBATCH" + __append_end_msg " Logstash Input Threads: $LSINPUTTHREADS" + __append_end_msg " Curator Day Cutoff: $CURCLOSEDAYS days" + __append_end_msg " Elasticsearch Storage Space: ${log_size_limit}GB" + else + __append_end_msg "Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE" + __append_end_msg "Logstash Heap Size: $NODE_LS_HEAP_SIZE" + __append_end_msg "Logstash Worker Count: $LSPIPELINEWORKERS" + __append_end_msg "Logstash Batch Size: $LSPIPELINEBATCH" + __append_end_msg "Logstash Input Threads: $LSINPUTTHREADS" + __append_end_msg "Curator Close After: $CURCLOSEDAYS days" + __append_end_msg "Elasticsearch Storage Space: ${log_size_limit}GB" + fi + + + # ADVANCED + if [[ $MANAGERADV == 'ADVANCED' ]]; then + __append_end_msg "Advanced Manager Settings:" + __append_end_msg " ES Cluster Name: $ESCLUSTERNAME" + if [[ ${#BLOGS[@]} -gt 0 ]]; then + __append_end_msg " Zeek Logs Enabled:" + for log in "${BLOGS[@]}"; do + __append_end_msg " - $log" + done + fi + fi + + if [[ $NSMSETUP == 'ADVANCED' ]]; then + __append_end_msg "Advanced NSM Settings:" + if [[ ${#ZEEKPINS[@]} -gt 0 ]]; then + local zeek_pin_str + for core in "${ZEEKPINS[@]}"; do + zeek_pin_str="${zeek_pin_str}${core}," + done + zeek_pin_str=${zeek_pin_str%,} + __append_end_msg " Zeek Pinned Cores: ${zeek_pin_str}" + fi + if [[ ${#SURIPINS[@]} -gt 0 ]]; then + local suri_pin_str + for core in "${SURIPINS[@]}"; do + suri_pin_str="${suri_pin_str}${core}," + done + suri_pin_str=${suri_pin_str%,} + __append_end_msg " Suricata Pinned Cores: ${suri_pin_str}" + fi + else + [[ -n $BASICZEEK ]] && __append_end_msg " Zeek Processes: $BASICZEEK" + [[ -n $BASICSURI ]] && __append_end_msg " Suricata Processes: $BASICSURI" + fi + + whiptail --yesno "$end_msg" 24 75 --scrolltext + local exitstatus=$? + whiptail_check_exitstatus +} + +__append_end_msg() { + local newline=$1 + + read -r -d '' end_msg <<- EOM + $end_msg + $newline + EOM +} + whiptail_eval_adv() { [ -n "$TESTING" ] && return @@ -1491,6 +1678,22 @@ whiptail_so_allow() { whiptail_check_exitstatus $exitstatus } +whitpail_ssh_warning() { + [ -n "$TESTING" ] && return + + local msg + + read -r -d '' msg <<- EOM + NOTE: You will recceive a warning upon SSH reconnect that the host key has changed. + + This is expected due to hardening of the OpenSSH server config. + + The host key algorithm will now be ED25519, follow the instructions given by your SSH client to remove the old key fingerprint then retry the connection. + EOM + + whiptail --msgbox "$msg" 14 75 +} + whiptail_storage_requirements() { local mount=$1 local current_val=$2 From 6d6829ba340a86d9ef2150c34765c305dd1d1558 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 6 Apr 2021 13:21:07 -0400 Subject: [PATCH 100/258] Remove duplicate variable assignment --- setup/so-setup | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 509ad419d..5b1a7417c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -558,7 +558,6 @@ if [[ $is_node && ! $is_eval ]]; then LSPIPELINEWORKERS=$num_cpu_cores LSPIPELINEBATCH=125 LSINPUTTHREADS=1 - LSPIPELINEBATCH=125 fi fi From af6403f8746180fdd3c94e1223dff3760a4acadd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 15:45:05 -0400 Subject: [PATCH 101/258] soup salt and repos ohh my --- salt/common/init.sls | 6 ++++ salt/common/tools/sbin/soup | 28 +++++++++++++++++-- salt/common/yum_repos/securityonion.repo | 14 +++++----- salt/common/yum_repos/securityonioncache.repo | 4 +-- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 0ada77e1a..adf34a43a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -74,6 +74,12 @@ repair_yumdb: - onlyif: - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' +crsynckeys: + file.recurse: + - name: /etc/pki/rpm_gpg + - source: salt://common/keys/ + + crbase: file.absent: - name: /etc/yum.repos.d/CentOS-Base.repo diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index cb2d19aed..a24af62f9 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -409,6 +409,30 @@ up_2.3.2X_to_2.3.30() { sed -i "/^strelka:/a \\ repos: \n - https://github.com/Neo23x0/signature-base" /opt/so/saltstack/local/pillar/global.sls; fi check_log_size_limit + INSTALLEDVERSION=2.3.30 +} + +up_2.3.3X_to_2.3.50() { + if [[ $OS == 'centos' ]]; then + # Import GPG Keys + gpg_rpm_import + + if [[ ! $is_airgap ]]; then + + DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') + + for DELREPO in "${DELREPOS[@]}"; + rm /etc/yum.repos.d/$DELREPO + done + + # Copy the new repo file if not airgap + cp $UPDATE_DIR/salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + yum clean all + yum repolist + fi + fi + INSTALLEDVERSION=2.3.50 + } verify_upgradespace() { @@ -503,7 +527,7 @@ upgrade_salt() { echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." echo "" # If CentOS - if [ "$OS" == "centos" ]; then + if [[ $OS == 'centos' ]]; then echo "Removing yum versionlock for Salt." echo "" yum versionlock delete "salt-*" @@ -518,7 +542,7 @@ upgrade_salt() { echo "" yum versionlock add "salt-*" # Else do Ubuntu things - elif [ "$OS" == "ubuntu" ]; then + elif [[ $OS == 'ubuntu' ]]; then echo "Removing apt hold for Salt." echo "" apt-mark unhold "salt-common" diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index e61829380..0cd96bd91 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -31,25 +31,25 @@ name=Extra Packages for Enterprise Linux 7 - $basearch baseurl=https://repo.securityonion.net/file/securityonion-repo/epel/ enabled=1 gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/RPM-GPG-KEY-EPEL-7 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7 [docker-ce-stable] name=Docker CE Stable - $basearch baseurl=https://repo.securityonion.net/file/securityonion-repo/docker-ce-stable enabled=1 gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub +gpgkey=file:///etc/pki/rpm-gpg/docker.pub -[saltstack] +[saltstack3003] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +gpgkey=file:///etc/pki/rpm-gpg/SALTSTACK-GPG-KEY.pub [wazuh_repo] gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh_repo/ @@ -57,7 +57,7 @@ protect=1 [wazuh4_repo] gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..bc0454ae7 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -40,9 +40,9 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub -[saltstack] +[saltstack3003] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From 92768ecd08e51d006064faf0beaf5081c3c0ad74 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 15:47:50 -0400 Subject: [PATCH 102/258] Add upgrade function --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index a24af62f9..6578432fa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -239,6 +239,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 + [[ "$INSTALLEDVERSION" == 2.3.30 ]] && up_2.3.3X_to_2.3.50 } postupgrade_changes() { From b50700114c4ee6a1ff0edbbd04419b6a447d91c1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 15:58:08 -0400 Subject: [PATCH 103/258] Add the do --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6578432fa..412f10e04 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -422,8 +422,8 @@ up_2.3.3X_to_2.3.50() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - for DELREPO in "${DELREPOS[@]}"; - rm /etc/yum.repos.d/$DELREPO + for DELREPO in "${DELREPOS[@]}"; do + rm /etc/yum.repos.d/"$DELREPO" done # Copy the new repo file if not airgap From 73e00dbe30dceeb5418932f49576643fbccdc2c9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 16:07:08 -0400 Subject: [PATCH 104/258] change salt upgrade in soup --- salt/common/tools/sbin/soup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c95f71499..a3fa70397 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -534,7 +534,11 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" + if [ $is_airgap -eq 0 ]; then + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + else + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + fi echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 51bf988d31e00fa61f8228460b3e087e28d93088 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 16:21:19 -0400 Subject: [PATCH 105/258] Add .repo extension --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 412f10e04..b4cbc43ec 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -423,7 +423,7 @@ up_2.3.3X_to_2.3.50() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in "${DELREPOS[@]}"; do - rm /etc/yum.repos.d/"$DELREPO" + rm /etc/yum.repos.d/"$DELREPO".repo done # Copy the new repo file if not airgap From 6f7e6cee80bae73ef6c7f4d68124f6a2f0828eab Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 16:43:42 -0400 Subject: [PATCH 106/258] Force it --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b4cbc43ec..46aa0c650 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -423,7 +423,7 @@ up_2.3.3X_to_2.3.50() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in "${DELREPOS[@]}"; do - rm /etc/yum.repos.d/"$DELREPO".repo + rm -f /etc/yum.repos.d/"$DELREPO".repo done # Copy the new repo file if not airgap From 099ac2ff19a86d2ffbf658a60b42266ba8e868aa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 09:06:22 -0400 Subject: [PATCH 107/258] Minor formatting changes to whiptail end screen --- setup/so-whiptail | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 45b263f96..e81c0be7c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -429,8 +429,6 @@ whiptail_end_settings() { # BASIC INFO (NETWORK, HOSTNAME, DESCRIPTION, ETC) read -r -d '' end_msg <<- EOM - The following options have been set, would you like to proceed? - Node Type: $install_type Hostname: $HOSTNAME EOM @@ -539,6 +537,11 @@ whiptail_end_settings() { done fi + if [[ $NSMSETUP != 'ADVANCED' ]]; then + [[ -n $BASICZEEK ]] && __append_end_msg "Zeek Processes: $BASICZEEK" + [[ -n $BASICSURI ]] && __append_end_msg "Suricata Processes: $BASICSURI" + fi + # ADVANCED OR REGULAR if [[ $NODESETUP == 'NODEADVANCED' ]]; then @@ -591,14 +594,12 @@ whiptail_end_settings() { suri_pin_str=${suri_pin_str%,} __append_end_msg " Suricata Pinned Cores: ${suri_pin_str}" fi - else - [[ -n $BASICZEEK ]] && __append_end_msg " Zeek Processes: $BASICZEEK" - [[ -n $BASICSURI ]] && __append_end_msg " Suricata Processes: $BASICSURI" fi - whiptail --yesno "$end_msg" 24 75 --scrolltext + whiptail --title "The following options have been set, would you like to proceed?" --yesno "$end_msg" 24 75 --scrolltext + local exitstatus=$? - whiptail_check_exitstatus + whiptail_check_exitstatus $exitstatus } __append_end_msg() { From 8f208728dde9fe1a4ef795da31c81357a60d41ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 09:10:16 -0400 Subject: [PATCH 108/258] change delete repos --- salt/common/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 5ce8d78c1..ffad5ad67 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -418,11 +418,11 @@ up_2.3.3X_to_2.3.50() { # Import GPG Keys gpg_rpm_import - if [[ ! $is_airgap ]]; then + if [ $is_airgap -eq 1 ]; then - DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') + DELREPOS=('CentOS-Base.repo' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - for DELREPO in "${DELREPOS[@]}"; do + for DELREPO in ${DELREPOS[@]}; do rm -f /etc/yum.repos.d/"$DELREPO".repo done From 5cd7d65b3f9c3d00115d4a181f86a4901abd1eff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 10:03:33 -0400 Subject: [PATCH 109/258] Fix Logic for Airgap distributed --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 214896572..9fef19875 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,7 +2285,7 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" ]]; then + if [[ "$OS" == "centos" && ! $is_airgap ]]; then mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ rm -f /etc/yum.repos.d/* From 88c565feae5934fe5a327201ec570188c802559c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 10:14:16 -0400 Subject: [PATCH 110/258] Fix proxy test logic --- setup/so-functions | 4 +++- setup/so-whiptail | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 8a751a4ad..e2e779775 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -563,7 +563,7 @@ collect_patch_schedule_name_import() { collect_proxy() { [[ -n $TESTING ]] && return - collect_proxy_details + collect_proxy_details || return while ! proxy_validate; do if whiptail_invalid_proxy; then collect_proxy_details no_ask @@ -608,6 +608,8 @@ collect_proxy_details() { so_proxy="$proxy_addr" fi export so_proxy + else + return 1 fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index e81c0be7c..fddf3b0fa 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -391,6 +391,7 @@ whiptail_dockernet_net() { whiptail_check_exitstatus $exitstatus } + whiptail_enable_components() { [ -n "$TESTING" ] && return From 5b3014496bdd493211bfd376f26dc91cddb1eece Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 10:35:59 -0400 Subject: [PATCH 111/258] Proxy fixes * Adjust proxy test timeout * Don't show proxy on error * Add echo statement so user knows what setup is doing --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e2e779775..0724f5851 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1872,12 +1872,13 @@ print_salt_state_apply() { } proxy_validate() { + echo "Testing proxy..." local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" - proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" 2>&1) + proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" --connect-timeout 5 2>&1) # set short connection timeout so user doesn't sit waiting for proxy test to timeout local ret=$? if [[ $ret != 0 ]]; then - error "Could not reach $test_url using proxy $so_proxy" + error "Could not reach $test_url using proxy provided" error "Received error: $proxy_test_err" if [[ -n $TESTING ]]; then error "Exiting setup" From 080ecba8e6e87700017f4ca365ff43db4bd62149 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 10:54:46 -0400 Subject: [PATCH 112/258] change delrepos --- salt/common/tools/sbin/soup | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ffad5ad67..0fc1017f6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -414,16 +414,18 @@ up_2.3.2X_to_2.3.30() { } up_2.3.3X_to_2.3.50() { - if [[ $OS == 'centos' ]]; then + if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import if [ $is_airgap -eq 1 ]; then - DELREPOS=('CentOS-Base.repo' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - + DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') + for DELREPO in ${DELREPOS[@]}; do - rm -f /etc/yum.repos.d/"$DELREPO".repo + if [[ -f "$DELREPO" ]]; then + rm -f /etc/yum.repos.d/"$DELREPO".repo + fi done # Copy the new repo file if not airgap @@ -433,7 +435,6 @@ up_2.3.3X_to_2.3.50() { fi fi INSTALLEDVERSION=2.3.50 - } verify_upgradespace() { From 54e039477623668ee5dc7a7858d13db191f6aca6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 10:57:09 -0400 Subject: [PATCH 113/258] change from saltstack3003 to just saltstack for repo --- salt/common/yum_repos/securityonion.repo | 4 ++-- salt/common/yum_repos/securityonioncache.repo | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index 0cd96bd91..9bb42552d 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -40,9 +40,9 @@ enabled=1 gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/docker.pub -[saltstack3003] +[saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ enabled=1 gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index bc0454ae7..6d5058337 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -40,9 +40,9 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub -[saltstack3003] +[saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From b7aa9ddaa3542c1c1d3724beac3444b7aebb0427 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 11:37:55 -0400 Subject: [PATCH 114/258] run preupgrade changes if 2.3.40 --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0fc1017f6..679c52a17 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -239,7 +239,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 - [[ "$INSTALLEDVERSION" == 2.3.30 ]] && up_2.3.3X_to_2.3.50 + [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50 } postupgrade_changes() { From ed0cd97de5af75600c3a4a2b2d95539de6378bca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 12:34:23 -0400 Subject: [PATCH 115/258] Fix Logic for Airgap distributed --- setup/so-functions | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 9fef19875..2e7a21797 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,15 +2285,17 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" && ! $is_airgap ]]; then - mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* - if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ - else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ - fi + if [[ "$OS" == "centos" ]]; then + if [[ ! $is_airgap ]]; then + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + fi + fi else echo "This is Ubuntu" fi From c8c1553247d6c07e441e5c62c6c8bced81f021dd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 12:36:50 -0400 Subject: [PATCH 116/258] Fix Logic for Airgap distributed --- setup/so-functions | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 2e7a21797..9aca9ca54 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,17 +2285,15 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" ]]; then - if [[ ! $is_airgap ]]; then - mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* - if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ - else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ - fi - fi + if [[ "$OS" == "centos" && $is_airgap ]]; then + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + fi else echo "This is Ubuntu" fi From 5578206bf119311a7b2ea7b99f8a0974b0054f81 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 12:41:01 -0400 Subject: [PATCH 117/258] need to make the repo changes before we try to upgrade sa;t --- salt/common/tools/sbin/soup | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 679c52a17..e9f337f65 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -230,6 +230,13 @@ masterunlock() { fi } +preupgrade_changes_2.3.50_repo() { + # We made repo changes in 2.3.50 and this prepares for that on upgrade + echo "Checking to see if 2.3.50 repo changes are needed." + + [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo +} + preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." @@ -413,7 +420,7 @@ up_2.3.2X_to_2.3.30() { INSTALLEDVERSION=2.3.30 } -up_2.3.3X_to_2.3.50() { +up_2.3.3X_to_2.3.50_repo() { if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import @@ -434,6 +441,9 @@ up_2.3.3X_to_2.3.50() { yum repolist fi fi +} + +up_2.3.3X_to_2.3.50() { INSTALLEDVERSION=2.3.50 } @@ -655,6 +665,7 @@ else update_registry update_docker_containers "soup" fi + echo "" echo "Stopping Salt Minion service." systemctl stop salt-minion @@ -665,6 +676,8 @@ echo "Stopping Salt Master service." systemctl stop salt-master echo "" +preupgrade_changes_2.3.50_repo + # Does salt need upgraded. If so update it. if [ "$UPGRADESALT" == "1" ]; then echo "Upgrading Salt" From 249fa06fc7f1c51475c7419dc86386d43d1e3639 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 13:03:27 -0400 Subject: [PATCH 118/258] echo when performing the repo actions for 2.3.50 --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e9f337f65..4584c207a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -421,12 +421,13 @@ up_2.3.2X_to_2.3.30() { } up_2.3.3X_to_2.3.50_repo() { + echo 'Performing 2.3.50 repo actions.' if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import if [ $is_airgap -eq 1 ]; then - + echo 'Deleting unneeded repo files.' DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in ${DELREPOS[@]}; do From ceb1ea61dcf25f3e9dfc021b11d0ff3e53240b40 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:15:49 -0400 Subject: [PATCH 119/258] Summary screen changes --- setup/so-whiptail | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index fddf3b0fa..eccf8c69c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -460,6 +460,7 @@ whiptail_end_settings() { for nic in "${BNICS[@]}"; do __append_end_msg " - $nic" done + __append_end_msg "MTU: $MTU" fi local homenet_arr @@ -506,20 +507,25 @@ whiptail_end_settings() { # PATCH SCHEDULE - [[ -n $PATCHSCHEDULENAME ]] && __append_end_msg "Patch schedule: $PATCHSCHEDULENAME" - - if [[ ${#PATCHSCHEDULEDAYS[@]} -gt 0 ]]; then - __append_end_msg "Day(s):" - for day in "${PATCHSCHEDULEDAYS[@]}"; do - __append_end_msg " - $day" - done - fi - - if [[ ${#PATCHSCHEDULEHOURS[@]} -gt 0 ]]; then - __append_end_msg "Hours(s):" - for hour in "${PATCHSCHEDULEHOURS[@]}"; do - __append_end_msg " - $hour" - done + if [[ -n $PATCHSCHEDULENAME ]]; then + __append_end_msg "Patch Schedule:" + if [[ $PATCHSCHEDULENAME != 'auto' && $PATCHSCHEDULENAME != 'manual' ]]; then + __append_end_msg " Type: $PATCHSCHEDULENAME" + else + __append_end_msg " Name: $PATCHSCHEDULENAME" + fi + if [[ ${#PATCHSCHEDULEDAYS[@]} -gt 0 ]]; then + __append_end_msg " Day(s):" + for day in "${PATCHSCHEDULEDAYS[@]}"; do + __append_end_msg " - $day" + done + fi + if [[ ${#PATCHSCHEDULEHOURS[@]} -gt 0 ]]; then + __append_end_msg " Hours(s):" + for hour in "${PATCHSCHEDULEHOURS[@]}"; do + __append_end_msg " - $hour" + done + fi fi # MISC From 377b14ccb1dab47eb8d6743cf626d047551f99fa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:20:55 -0400 Subject: [PATCH 120/258] ESCLUSTERNAME is empty for standalone, so check if it's set before listing --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index eccf8c69c..2522e65f9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -574,7 +574,7 @@ whiptail_end_settings() { # ADVANCED if [[ $MANAGERADV == 'ADVANCED' ]]; then __append_end_msg "Advanced Manager Settings:" - __append_end_msg " ES Cluster Name: $ESCLUSTERNAME" + [[ -n $ESCLUSTERNAME ]] && __append_end_msg " ES Cluster Name: $ESCLUSTERNAME" if [[ ${#BLOGS[@]} -gt 0 ]]; then __append_end_msg " Zeek Logs Enabled:" for log in "${BLOGS[@]}"; do From 425e5bc4c30dda35c27d730a1c465120bc05821b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 13:31:43 -0400 Subject: [PATCH 121/258] add some quotes --- salt/common/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 4584c207a..6824ac8a2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -421,18 +421,19 @@ up_2.3.2X_to_2.3.30() { } up_2.3.3X_to_2.3.50_repo() { - echo 'Performing 2.3.50 repo actions.' + echo "Performing 2.3.50 repo actions." if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import if [ $is_airgap -eq 1 ]; then - echo 'Deleting unneeded repo files.' + echo "Deleting unneeded repo files." DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - for DELREPO in ${DELREPOS[@]}; do + for DELREPO in "${DELREPOS[@]}"; do if [[ -f "$DELREPO" ]]; then - rm -f /etc/yum.repos.d/"$DELREPO".repo + echo "Deleting $DELREPO.repo" + rm -f "/etc/yum.repos.d/$DELREPO.repo" fi done From f83ac5a2788089946ab5dc79a5bdb06e712bce04 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:38:47 -0400 Subject: [PATCH 122/258] Print install summary to file and setup log after user confirms --- setup/so-whiptail | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 2522e65f9..3e4a79a91 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -607,6 +607,9 @@ whiptail_end_settings() { local exitstatus=$? whiptail_check_exitstatus $exitstatus + + echo "$end_msg" > /root/install_summary + printf '%s\n' 'Install summary:' "$end_msg" >> "$setup_log" } __append_end_msg() { From ec076bba4ae58d11a92974b39d84cf44b80783e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:42:18 -0400 Subject: [PATCH 123/258] MTU is not always set by the user, so don't always show in summary --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 3e4a79a91..7c2665363 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -460,7 +460,7 @@ whiptail_end_settings() { for nic in "${BNICS[@]}"; do __append_end_msg " - $nic" done - __append_end_msg "MTU: $MTU" + [[ -n $MTU ]] && __append_end_msg "MTU: $MTU" fi local homenet_arr From c4f01192763e4f661eee745777a25f6802b1ed9a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 13:51:40 -0400 Subject: [PATCH 124/258] fix check if repo file exists --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6824ac8a2..6448c432f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -431,7 +431,7 @@ up_2.3.3X_to_2.3.50_repo() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in "${DELREPOS[@]}"; do - if [[ -f "$DELREPO" ]]; then + if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then echo "Deleting $DELREPO.repo" rm -f "/etc/yum.repos.d/$DELREPO.repo" fi From 3a4cf8aa269cc1dee056f27a4d54eb784fb5d1ae Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:54:01 -0400 Subject: [PATCH 125/258] Add proxy url/user to summary --- setup/so-whiptail | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 7c2665363..000aa2ba4 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -449,6 +449,14 @@ whiptail_end_settings() { __append_end_msg "Management NIC: $MNIC" __append_end_msg "Management IP: $MAINIP" + if [[ -n $so_proxy ]]; then + __append_end_msg "Proxy:" + __append_end_msg " Server URL: $proxy_addr" + [[ -n $proxy_user ]] && __append_end_msg " User: $proxy_user" + else + __append_end_msg "Proxy: N/A" + fi + if [[ $address_type == 'STATIC' ]]; then __append_end_msg "Gateway: $MGATEWAY" __append_end_msg "DNS: $MDNS" From 3c69c0c24cbb501e628dc40ab00cf50092589298 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 14:15:02 -0400 Subject: [PATCH 126/258] Correct patch schedule name logic in summary --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 000aa2ba4..33053a273 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -517,7 +517,7 @@ whiptail_end_settings() { if [[ -n $PATCHSCHEDULENAME ]]; then __append_end_msg "Patch Schedule:" - if [[ $PATCHSCHEDULENAME != 'auto' && $PATCHSCHEDULENAME != 'manual' ]]; then + if [[ $PATCHSCHEDULENAME == 'auto'|| $PATCHSCHEDULENAME == 'manual' ]]; then __append_end_msg " Type: $PATCHSCHEDULENAME" else __append_end_msg " Name: $PATCHSCHEDULENAME" From 9baa9767cafb722fb864bfd90e27e2622f6b497d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 16:12:51 -0400 Subject: [PATCH 127/258] Add raid bind --- salt/telegraf/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index 2814eb159..c4871a0b3 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -72,6 +72,7 @@ so-telegraf: - /opt/so/conf/telegraf/scripts:/scripts:ro - /opt/so/log/stenographer:/var/log/stenographer:ro - /opt/so/log/suricata:/var/log/suricata:ro + - /opt/so/log/raid:/var/log/raid:ro - watch: - file: tgrafconf - file: tgrafsyncscripts From 8ab4dd10d42ac97b522abb2345d45e00ba36f43c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 16:29:44 -0400 Subject: [PATCH 128/258] Add sostatus for telegraf --- salt/common/init.sls | 19 +++++++++++++++++++ salt/telegraf/init.sls | 1 + 2 files changed, 20 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index 0ada77e1a..9ee126ac1 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -297,6 +297,25 @@ commonlogrotateconf: - month: '*' - dayweek: '*' +# Create the status directory +sostatusdir: + file.directory: + - name: /opt/so/log/sostatus + - user: 0 + - group: 0 + - makedirs: True + +# Install sostatus check cron +/usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: + cron.present: + - user: root + - minute: '*/15' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + + {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %} # Lock permissions on the backup directory backupdir: diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index c4871a0b3..cea4d3f45 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -73,6 +73,7 @@ so-telegraf: - /opt/so/log/stenographer:/var/log/stenographer:ro - /opt/so/log/suricata:/var/log/suricata:ro - /opt/so/log/raid:/var/log/raid:ro + - /opt/so/log/sostatus:/var/log/sostatus:ro - watch: - file: tgrafconf - file: tgrafsyncscripts From 3caaf0682043074b1576d175f3ec1ba94f95b6bb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 16:30:16 -0400 Subject: [PATCH 129/258] Add sostatus for telegraf --- salt/telegraf/scripts/sostatus.sh | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 salt/telegraf/scripts/sostatus.sh diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh new file mode 100644 index 000000000..23096d903 --- /dev/null +++ b/salt/telegraf/scripts/sostatus.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=sostatus +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf +SOSTATUSLOG=/var/log/sostatus/status.log +SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) + +if [ -f "$SOSTATUSLOG" ]; then + echo "sostatus status=$SOSTATUSSTATUS" +else + exit 0 +fi From a5f5888913e00ab59d3459701d01f37e6e357b1e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 17:03:08 -0400 Subject: [PATCH 130/258] Summary order change --- setup/so-whiptail | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 33053a273..493ae7a68 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -449,6 +449,12 @@ whiptail_end_settings() { __append_end_msg "Management NIC: $MNIC" __append_end_msg "Management IP: $MAINIP" + if [[ $address_type == 'STATIC' ]]; then + __append_end_msg "Gateway: $MGATEWAY" + __append_end_msg "DNS: $MDNS" + __append_end_msg "DNS Domain: $MSEARCH" + fi + if [[ -n $so_proxy ]]; then __append_end_msg "Proxy:" __append_end_msg " Server URL: $proxy_addr" @@ -457,12 +463,6 @@ whiptail_end_settings() { __append_end_msg "Proxy: N/A" fi - if [[ $address_type == 'STATIC' ]]; then - __append_end_msg "Gateway: $MGATEWAY" - __append_end_msg "DNS: $MDNS" - __append_end_msg "DNS Domain: $MSEARCH" - fi - if [[ $is_sensor ]]; then __append_end_msg "Bond NIC(s):" for nic in "${BNICS[@]}"; do From d4a3bc455071b90b36b9233fb7392b8fdbd4cfa9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 08:43:20 -0400 Subject: [PATCH 131/258] Fix so repo for salt --- salt/common/yum_repos/securityonion.repo | 7 +++++++ salt/common/yum_repos/securityonioncache.repo | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index e61829380..0f39d5a3f 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -47,6 +47,13 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +[saltstack3003] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + [wazuh_repo] gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..def6f8a40 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -47,6 +47,13 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +[saltstack3003] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + [wazuh_repo] gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH From dce476b604e30923ff79e566eea3593bbbdc6a30 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 8 Apr 2021 09:54:41 -0400 Subject: [PATCH 132/258] change back to saltstack3003 repo --- salt/common/yum_repos/securityonion.repo | 2 +- salt/common/yum_repos/securityonioncache.repo | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index 9bb42552d..5fd9b5fc4 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -42,7 +42,7 @@ gpgkey=file:///etc/pki/rpm-gpg/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..56ada1413 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -42,7 +42,7 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From 725320ebc8880629f3c2550a21e55b1ba3785e52 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:02:11 -0400 Subject: [PATCH 133/258] Fix Repo Logic --- salt/common/init.sls | 2 +- setup/so-functions | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 9ee126ac1..93f76c3b3 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -309,7 +309,7 @@ sostatusdir: /usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: cron.present: - user: root - - minute: '*/15' + - minute: '*/5' - hour: '*' - daymonth: '*' - month: '*' diff --git a/setup/so-functions b/setup/so-functions index 9aca9ca54..54f8d2abf 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,14 +2285,18 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" && $is_airgap ]]; then - mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* - if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + if [[ "$OS" == "centos" ]]; then + if [[ "$INTERWEBS" == "AIRGAP" ]]; then + echo "This is airgap I don't need to add this repo" else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + fi fi else echo "This is Ubuntu" From ce9f781d81ea51ad265b630e5f79f1959db161ab Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:24:04 -0400 Subject: [PATCH 134/258] Fix Repo Logic --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 54f8d2abf..1633d0901 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2291,7 +2291,7 @@ securityonion_repo() { else mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* + rm -rf /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else From 951369c2d698cbd9a7a738c839c099ffae55d430 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:25:36 -0400 Subject: [PATCH 135/258] Fix Repo Logic --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 1633d0901..0771567b2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2290,7 +2290,7 @@ securityonion_repo() { echo "This is airgap I don't need to add this repo" else mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ + mv -v /etc/yum.repos.d/* /root/oldrepos/ rm -rf /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ From fdaf251ba02111b21244524018d7dd6a1b13c6a4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:36:52 -0400 Subject: [PATCH 136/258] Fix Repo Logic --- setup/so-functions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 0771567b2..b8cd2cae2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2291,7 +2291,9 @@ securityonion_repo() { else mkdir -p /root/oldrepos mv -v /etc/yum.repos.d/* /root/oldrepos/ - rm -rf /etc/yum.repos.d/* + ls -la /etc/yum.repos.d/ + rm -rf /etc/yum.repos.d + mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else From 4c5f373ffa2c3e5f3532d32e42c1d50c1e88e6a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:37:44 -0400 Subject: [PATCH 137/258] Fix Repo Logic --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index b8cd2cae2..f2face572 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2293,6 +2293,7 @@ securityonion_repo() { mv -v /etc/yum.repos.d/* /root/oldrepos/ ls -la /etc/yum.repos.d/ rm -rf /etc/yum.repos.d + yum clean all mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ From 09b14e6a863ede8568d0b0622f3a18dd41af9eb1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:38:50 -0400 Subject: [PATCH 138/258] Fix Repo Logic --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index f2face572..9cbad1cfb 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2294,6 +2294,7 @@ securityonion_repo() { ls -la /etc/yum.repos.d/ rm -rf /etc/yum.repos.d yum clean all + yum repolist all mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ From f7f95b6c541c6db3a5264f1094da7b604b3ff617 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 8 Apr 2021 11:22:54 -0400 Subject: [PATCH 139/258] Add model to sensoroni agent config --- salt/sensoroni/files/sensoroni.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index df2990404..378d42373 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,9 +1,6 @@ {%- set URLBASE = salt['pillar.get']('global:url_base') %} -{%- if salt['pillar.get']('sensoroni:node_description') %} -{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %} -{%- else %} -{%- set DESCRIPTION = salt['grains.get']('sosmodel', '') %} -{%- endif %} +{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description', '') %} +{%- set MODEL = salt['grains.get']('sosmodel', '') %} {%- set ADDRESS = salt['pillar.get']('sensoroni:node_address') %} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} {%- set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) %} @@ -21,6 +18,7 @@ "role": "{{ grains.role }}", "description": "{{ DESCRIPTION }}", "address": "{{ ADDRESS }}", + "model": "{{ MODEL }}", "pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }}, "serverUrl": "https://{{ URLBASE }}/sensoroniagents", "verifyCert": false, From b53815d04af3731f59fdf466653f040715d10f7f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 11:42:41 -0400 Subject: [PATCH 140/258] Fix Telegraf sostatus --- salt/telegraf/etc/telegraf.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index f6bcbdaf5..1b172485b 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -663,6 +663,15 @@ # # Read metrics from one or more commands that can output to stdout +[[inputs.exec]] + commands = [ + "/scripts/sostatus.sh" + ] + data_format = "influx" + timeout = "15s" + interval = "180s" + + # ## Commands array {% if grains['role'] in ['so-manager', 'so-managersearch'] %} [[inputs.exec]] From 6650ad5cdd545e8378a945edf7abfd94b92f85d5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 8 Apr 2021 14:04:30 -0400 Subject: [PATCH 141/258] make the -r for all --- salt/common/tools/sbin/soup | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6448c432f..70d453334 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -547,11 +547,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - if [ $is_airgap -eq 0 ]; then - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" - else - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - fi + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From c9feda116853bf91d5160c8205dc4602268c6afe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 9 Apr 2021 08:48:29 -0400 Subject: [PATCH 142/258] Do not upgrade salt on ISO installs --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index b7acf7cb1..ba815e57f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2718,7 +2718,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then yum repolist >> /dev/null - yum -y update >> "$setup_log" + yum -y update --exclude=salt* >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 From b2fcd438c26d699da9a556fc7558c207caf6927a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 09:39:33 -0400 Subject: [PATCH 143/258] Initial support for checking state of manager during setup --- salt/common/init.sls | 5 +++++ setup/so-functions | 19 +++++++++++++++++++ setup/so-setup | 4 ++++ setup/so-whiptail | 17 +++++++++++++++-- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 93f76c3b3..3ce6286be 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -304,6 +304,11 @@ sostatusdir: - user: 0 - group: 0 - makedirs: True + +sostatus_log: + file.managed: + - name: /opt/so/log/sostatus/status.log + - mode: 644 # Install sostatus check cron /usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: diff --git a/setup/so-functions b/setup/so-functions index b7acf7cb1..604eb56b7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -162,6 +162,25 @@ check_hive_init() { docker rm so-thehive } +check_manager_state() { + echo "Checking state of manager services. This may take a moment..." + retry 2 15 "__check_so_status" && retry 2 15 "__check_salt_master" && return 0 || return 1 +} + +__check_so_status() { + local so_status_output + so_status_output=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /opt/so/log/sostatus/status.log) + [[ -z $so_status_output ]] && so_status_output=1 + return $so_status_output +} + +__check_salt_master() { + local salt_master_status + salt_master_status=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" systemctl is-active --quiet salt-master) + [[ -z $salt_master_status ]] && salt_master_status=1 + return $salt_master_status +} + check_network_manager_conf() { local gmdconf="/usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf" local nmconf="/etc/NetworkManager/NetworkManager.conf" diff --git a/setup/so-setup b/setup/so-setup index 5b1a7417c..79c9b9784 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -574,6 +574,10 @@ if [[ $is_manager || $is_import ]]; then collect_so_allow; fi # This block sets REDIRECTIT which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 +if [[ $is_minion ]] && ! check_manager_state; then + whiptail_manager_error || exit 1 +fi + whiptail_end_settings # From here on changes will be made. diff --git a/setup/so-whiptail b/setup/so-whiptail index 493ae7a68..2b1199fb5 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -79,7 +79,7 @@ whiptail_bond_nics_mtu() { whiptail_cancel() { - whiptail --title "Security Onion Setup" --msgbox "Cancelling Setup. No changes have been made." 8 75 + whiptail --title "Security Onion Setup" --msgbox "Cancelling Setup." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -88,7 +88,7 @@ whiptail_cancel() { } >> $setup_log 2>&1 fi - title "User cancelled setup, no changes made." + title "User cancelled setup." exit } @@ -1140,6 +1140,19 @@ whiptail_manager_adv_service_zeeklogs() { } +whiptail_manager_error() { + local msg + read -r -d '' <<- EOM + Setup could not determine if the manager $MSRV is in a good state. + + Continuing without verifying all services on the manager are running may result in a failure. + + Would you like to continue anyway? + EOM + + whiptail --title "Security Onion Setup" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 +} + whiptail_manager_updates() { [ -n "$TESTING" ] && return From 9b5276f1ab401a65f99de480304d8bfd1174ec8a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 09:59:54 -0400 Subject: [PATCH 144/258] Remove bad `||` statement --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 79c9b9784..84fcf5c34 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -575,7 +575,7 @@ if [[ $is_manager || $is_import ]]; then collect_so_allow; fi set_redirect >> $setup_log 2>&1 if [[ $is_minion ]] && ! check_manager_state; then - whiptail_manager_error || exit 1 + whiptail_manager_error fi whiptail_end_settings From fc9df2bbaec84bc8d6a1d780074de4cd588f0019 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:00:50 -0400 Subject: [PATCH 145/258] Update airgap question to ask during minion installs too --- setup/so-setup | 2 +- setup/so-whiptail | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 84fcf5c34..f53909bdc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -264,7 +264,7 @@ elif [ "$install_type" = 'ANALYST' ]; then fi # Check if this is an airgap install -if [[ ( $is_manager || $is_import ) && $is_iso ]]; then +if [[ $is_iso || $is_minion ]]; then whiptail_airgap if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true diff --git a/setup/so-whiptail b/setup/so-whiptail index 2b1199fb5..368b6b0cd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -19,10 +19,13 @@ whiptail_airgap() { [ -n "$TESTING" ] && return + local node_str='node' + [[ $is_manager || $is_import ]] && node_str='manager' + INTERWEBS=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose your install conditions:" 20 75 4 \ - "STANDARD" "This manager has internet accesss" ON \ - "AIRGAP" "This manager does not have internet access" OFF 3>&1 1>&2 2>&3 ) + "STANDARD" "This $node_str has internet accesss" ON \ + "AIRGAP" "This $node_str does not have internet access" OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 764307bfa055a667f00b8fd5fae5b173c5407103 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:09:28 -0400 Subject: [PATCH 146/258] Reformat airgap whiptail prompt --- setup/so-whiptail | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 368b6b0cd..1fbfa34dd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -23,12 +23,14 @@ whiptail_airgap() { [[ $is_manager || $is_import ]] && node_str='manager' INTERWEBS=$(whiptail --title "Security Onion Setup" --radiolist \ - "Choose your install conditions:" 20 75 4 \ - "STANDARD" "This $node_str has internet accesss" ON \ - "AIRGAP" "This $node_str does not have internet access" OFF 3>&1 1>&2 2>&3 ) + "How should this $node_str be installed?" 10 60 2 \ + "Standard " "This $node_str has internet accesss." ON \ + "Airgap " "This $node_str does not have internet access." OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus + + INTERWEBS=$(echo "${INTERWEBS^^}" | tr -d ' ') } whiptail_avoid_default_hostname() { From 026ce769666f11a4e439dc70149654863a0cd1e9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:11:00 -0400 Subject: [PATCH 147/258] Change airgap prompt to menu --- setup/so-whiptail | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 1fbfa34dd..50b50e353 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -22,10 +22,10 @@ whiptail_airgap() { local node_str='node' [[ $is_manager || $is_import ]] && node_str='manager' - INTERWEBS=$(whiptail --title "Security Onion Setup" --radiolist \ + INTERWEBS=$(whiptail --title "Security Onion Setup" --menu \ "How should this $node_str be installed?" 10 60 2 \ - "Standard " "This $node_str has internet accesss." ON \ - "Airgap " "This $node_str does not have internet access." OFF 3>&1 1>&2 2>&3 ) + "Standard " "This $node_str has internet accesss" \ + "Airgap " "This $node_str does not have internet access" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 4db20a00fff56b2be682cb4199c975acfb83dfc7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:16:19 -0400 Subject: [PATCH 148/258] Add quotes around description, since it can contain spaces --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index f53909bdc..bac2fcdfd 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -339,7 +339,7 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ "MSRVIP=$MSRVIP" \ - "NODE_DESCRIPTION=$NODE_DESCRIPTION" > "$install_opt_file" + "NODE_DESCRIPTION=\"$NODE_DESCRIPTION\"" > "$install_opt_file" [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" From 5cb73ced36ef40f6683be14db1d5ec4391fd8328 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 9 Apr 2021 14:58:15 -0400 Subject: [PATCH 149/258] Add Influx module to SOC config --- salt/soc/files/soc/soc.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 6e2850aca..1dcd46c63 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -53,6 +53,13 @@ "cacheMs": {{ ES_FIELDCAPS_CACHE }}, "verifyCert": false }, + "influxdb": { + "hostUrl": "https://{{ MANAGERIP }}:8086", + "token": "", + "org": "", + "bucket": "telegraf", + "verifyCert": false + }, "sostatus": { "refreshIntervalMs": 30000, "offlineThresholdMs": 900000 From 8facbcf18c16b432fdb2beef12cbfc023c8abead Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 9 Apr 2021 20:40:44 -0400 Subject: [PATCH 150/258] Do not set influxdb hostUrl if import node since import nodes don't run influxdb --- salt/soc/files/soc/soc.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 1dcd46c63..6f1c3a6da 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -54,7 +54,11 @@ "verifyCert": false }, "influxdb": { +{%- if grains['role'] in ['so-import'] %} + "hostUrl": "", +{%- else %} "hostUrl": "https://{{ MANAGERIP }}:8086", +{%- endif %} "token": "", "org": "", "bucket": "telegraf", From 5d98c896a3791434491afdedc1de6a55fbf87a40 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 12:53:17 -0400 Subject: [PATCH 151/258] /opt/so/log needs 755 permissions for soremote to read sostatus log --- salt/common/init.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index 3ce6286be..ee7cad5e6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -51,6 +51,11 @@ sosaltstackperms: - gid: 939 - dir_mode: 770 +so_log_perms: + file.directory: + - name: /opt/so/log + - dir_mode: 755 + # Create a state directory statedir: file.directory: From 73a1bdd885f7943ee52538e025282dcc736101fa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 12:59:45 -0400 Subject: [PATCH 152/258] Send stdout to log, and actually populate error message --- setup/so-functions | 2 +- setup/so-whiptail | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 604eb56b7..5f07bb3b0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -164,7 +164,7 @@ check_hive_init() { check_manager_state() { echo "Checking state of manager services. This may take a moment..." - retry 2 15 "__check_so_status" && retry 2 15 "__check_salt_master" && return 0 || return 1 + retry 2 15 "__check_so_status" >> $setup_log 2>&1 && retry 2 15 "__check_salt_master" >> $setup_log 2>&1 && return 0 || return 1 } __check_so_status() { diff --git a/setup/so-whiptail b/setup/so-whiptail index 50b50e353..6ce2c214c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1147,7 +1147,7 @@ whiptail_manager_adv_service_zeeklogs() { whiptail_manager_error() { local msg - read -r -d '' <<- EOM + read -r -d '' msg <<- EOM Setup could not determine if the manager $MSRV is in a good state. Continuing without verifying all services on the manager are running may result in a failure. From 9240d376f3588d4029b14ff93400c2031bde9cf7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 12 Apr 2021 14:31:41 -0400 Subject: [PATCH 153/258] combine client repo management into 1 state --- salt/airgap/init.sls | 71 ---------------- salt/common/init.sls | 83 ------------------- .../client/files/centos/airgap}/yum.conf | 0 .../client/files/centos}/keys/GPG-KEY-WAZUH | 0 .../files/centos}/keys/RPM-GPG-KEY-EPEL-7 | 0 .../files/centos}/keys/SALTSTACK-GPG-KEY.pub | 0 .../client/files/centos}/keys/docker.pub | 0 .../files/centos}/keys/securityonion.pub | 0 .../client/files/centos}/securityonion.repo | 0 .../files/centos}/securityonioncache.repo | 0 .../client/files/centos}/yum.conf.jinja | 0 salt/repo/client/init.sls | 77 +++++++++++++++++ salt/repo/client/map.jinja | 25 ++++++ salt/top.sls | 11 +-- salt/yum/init.sls | 17 ---- 15 files changed, 104 insertions(+), 180 deletions(-) delete mode 100644 salt/airgap/init.sls rename salt/{airgap/files => repo/client/files/centos/airgap}/yum.conf (100%) rename salt/{common => repo/client/files/centos}/keys/GPG-KEY-WAZUH (100%) rename salt/{common => repo/client/files/centos}/keys/RPM-GPG-KEY-EPEL-7 (100%) rename salt/{common => repo/client/files/centos}/keys/SALTSTACK-GPG-KEY.pub (100%) rename salt/{common => repo/client/files/centos}/keys/docker.pub (100%) rename salt/{common => repo/client/files/centos}/keys/securityonion.pub (100%) rename salt/{common/yum_repos => repo/client/files/centos}/securityonion.repo (100%) rename salt/{common/yum_repos => repo/client/files/centos}/securityonioncache.repo (100%) rename salt/{yum/etc => repo/client/files/centos}/yum.conf.jinja (100%) create mode 100644 salt/repo/client/init.sls create mode 100644 salt/repo/client/map.jinja delete mode 100644 salt/yum/init.sls diff --git a/salt/airgap/init.sls b/salt/airgap/init.sls deleted file mode 100644 index 4ff401099..000000000 --- a/salt/airgap/init.sls +++ /dev/null @@ -1,71 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - -{% set MANAGER = salt['grains.get']('master') %} -airgapyum: - file.managed: - - name: /etc/yum/yum.conf - - source: salt://airgap/files/yum.conf - -airgap_repo: - pkgrepo.managed: - - humanname: Airgap Repo - - baseurl: https://{{ MANAGER }}/repo - - gpgcheck: 1 - - sslverify: 0 - -agbase: - file.absent: - - name: /etc/yum.repos.d/CentOS-Base.repo - -agcr: - file.absent: - - name: /etc/yum.repos.d/CentOS-CR.repo - -agdebug: - file.absent: - - name: /etc/yum.repos.d/CentOS-Debuginfo.repo - -agfasttrack: - file.absent: - - name: /etc/yum.repos.d/CentOS-fasttrack.repo - -agmedia: - file.absent: - - name: /etc/yum.repos.d/CentOS-Media.repo - -agsources: - file.absent: - - name: /etc/yum.repos.d/CentOS-Sources.repo - -agvault: - file.absent: - - name: /etc/yum.repos.d/CentOS-Vault.repo - -agkernel: - file.absent: - - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo - -agepel: - file.absent: - - name: /etc/yum.repos.d/epel.repo - -agtesting: - file.absent: - - name: /etc/yum.repos.d/epel-testing.repo - -agssrepo: - file.absent: - - name: /etc/yum.repos.d/saltstack.repo - -agwazrepo: - file.absent: - - name: /etc/yum.repos.d/wazuh.repo - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} \ No newline at end of file diff --git a/salt/common/init.sls b/salt/common/init.sls index b630891f5..0492b6535 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -2,8 +2,6 @@ {% if sls in allowed_states %} {% set role = grains.id.split('_') | last %} -{% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -66,87 +64,6 @@ salttmp: - group: 939 - makedirs: True -# Remove default Repos -{% if grains['os'] == 'CentOS' %} -repair_yumdb: - cmd.run: - - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all' - - onlyif: - - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' - -crsynckeys: - file.recurse: - - name: /etc/pki/rpm_gpg - - source: salt://common/keys/ - - -crbase: - file.absent: - - name: /etc/yum.repos.d/CentOS-Base.repo - -crcr: - file.absent: - - name: /etc/yum.repos.d/CentOS-CR.repo - -crdebug: - file.absent: - - name: /etc/yum.repos.d/CentOS-Debuginfo.repo - -crdockerce: - file.absent: - - name: /etc/yum.repos.d/docker-ce.repo - -crfasttrack: - file.absent: - - name: /etc/yum.repos.d/CentOS-fasttrack.repo - -crmedia: - file.absent: - - name: /etc/yum.repos.d/CentOS-Media.repo - -crsources: - file.absent: - - name: /etc/yum.repos.d/CentOS-Sources.repo - -crvault: - file.absent: - - name: /etc/yum.repos.d/CentOS-Vault.repo - -crkernel: - file.absent: - - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo - -crepel: - file.absent: - - name: /etc/yum.repos.d/epel.repo - -crtesting: - file.absent: - - name: /etc/yum.repos.d/epel-testing.repo - -crssrepo: - file.absent: - - name: /etc/yum.repos.d/saltstack.repo - -crwazrepo: - file.absent: - - name: /etc/yum.repos.d/wazuh.repo - -{% if not ISAIRGAP %} -crsecurityonionrepo: - file.managed: - {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - - name: /etc/yum.repos.d/securityonion.repo - - source: salt://common/yum_repos/securityonion.repo - {% else %} - - name: /etc/yum.repos.d/securityonioncache.repo - - source: salt://common/yum_repos/securityonioncache.repo - {% endif %} - - mode: 644 - -{% endif %} -{% endif %} - # Install common packages {% if grains['os'] != 'CentOS' %} commonpkgs: diff --git a/salt/airgap/files/yum.conf b/salt/repo/client/files/centos/airgap/yum.conf similarity index 100% rename from salt/airgap/files/yum.conf rename to salt/repo/client/files/centos/airgap/yum.conf diff --git a/salt/common/keys/GPG-KEY-WAZUH b/salt/repo/client/files/centos/keys/GPG-KEY-WAZUH similarity index 100% rename from salt/common/keys/GPG-KEY-WAZUH rename to salt/repo/client/files/centos/keys/GPG-KEY-WAZUH diff --git a/salt/common/keys/RPM-GPG-KEY-EPEL-7 b/salt/repo/client/files/centos/keys/RPM-GPG-KEY-EPEL-7 similarity index 100% rename from salt/common/keys/RPM-GPG-KEY-EPEL-7 rename to salt/repo/client/files/centos/keys/RPM-GPG-KEY-EPEL-7 diff --git a/salt/common/keys/SALTSTACK-GPG-KEY.pub b/salt/repo/client/files/centos/keys/SALTSTACK-GPG-KEY.pub similarity index 100% rename from salt/common/keys/SALTSTACK-GPG-KEY.pub rename to salt/repo/client/files/centos/keys/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/keys/docker.pub b/salt/repo/client/files/centos/keys/docker.pub similarity index 100% rename from salt/common/keys/docker.pub rename to salt/repo/client/files/centos/keys/docker.pub diff --git a/salt/common/keys/securityonion.pub b/salt/repo/client/files/centos/keys/securityonion.pub similarity index 100% rename from salt/common/keys/securityonion.pub rename to salt/repo/client/files/centos/keys/securityonion.pub diff --git a/salt/common/yum_repos/securityonion.repo b/salt/repo/client/files/centos/securityonion.repo similarity index 100% rename from salt/common/yum_repos/securityonion.repo rename to salt/repo/client/files/centos/securityonion.repo diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/repo/client/files/centos/securityonioncache.repo similarity index 100% rename from salt/common/yum_repos/securityonioncache.repo rename to salt/repo/client/files/centos/securityonioncache.repo diff --git a/salt/yum/etc/yum.conf.jinja b/salt/repo/client/files/centos/yum.conf.jinja similarity index 100% rename from salt/yum/etc/yum.conf.jinja rename to salt/repo/client/files/centos/yum.conf.jinja diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls new file mode 100644 index 000000000..60353426f --- /dev/null +++ b/salt/repo/client/init.sls @@ -0,0 +1,77 @@ +{% from 'repo/client/map.jinja' import ABSENTFILES with context %} +{% from 'repo/client/map.jinja' import REPOPATH with context %} +{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} +{% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} +{% set role = grains.id.split('_') | last %} + +# from airgap state +{% if ISAIRGAP and grains.os == 'CentOS' %} +{% set MANAGER = salt['grains.get']('master') %} +airgapyum: + file.managed: + - name: /etc/yum/yum.conf + - source: salt://repo/client/files/centos/airgap/yum.conf + +airgap_repo: + pkgrepo.managed: + - humanname: Airgap Repo + - baseurl: https://{{ MANAGER }}/repo + - gpgcheck: 1 + - sslverify: 0 +{% endif %} + +# from airgap and common +{% if ABSENTFILES|length > 0%} + {% for file in ABSENTFILES %} +{{ file }}: + file.absent: + - name: {{ REPOPATH }}{{ file }} + - onchanges_in: cleanyum + {% endfor %} +{% endif %} + +# from common state +# Remove default Repos +{% if grains['os'] == 'CentOS' %} +repair_yumdb: + cmd.run: + - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all' + - onlyif: + - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' + +crsynckeys: + file.recurse: + - name: /etc/pki/rpm_gpg + - source: salt://repo/client/files/centos/keys/ + +{% if not ISAIRGAP %} +crsecurityonionrepo: + file.managed: + {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} + - name: /etc/yum.repos.d/securityonion.repo + - source: salt://repo/client/files/centos/securityonion.repo + {% else %} + - name: /etc/yum.repos.d/securityonioncache.repo + - source: salt://repo/client/files/centos/securityonioncache.repo + {% endif %} + - mode: 644 + +yumconf: + file.managed: + - name: /etc/yum.conf + - source: salt:/repo/client/files/centos/yum.conf.jinja + - mode: 644 + - template: jinja +{% endif %} + +cleanyum: + module.run: + - pkg.clean_metadata + - onchanges: + - file: airgapyum + - pkgrepo: airgap_repo + - file: crsecurityonionrepo + - file: yumconf + +{% endif %} + diff --git a/salt/repo/client/map.jinja b/salt/repo/client/map.jinja new file mode 100644 index 000000000..ccfa1eae2 --- /dev/null +++ b/salt/repo/client/map.jinja @@ -0,0 +1,25 @@ +{% if grains.os == 'CentOS' %} + + {% set REPOPATH = '/etc/yum.repos.d/' %} + {% set ABSENTFILES = [ + 'CentOS-Base.repo', + 'CentOS-CR.repo', + 'CentOS-Debuginfo.repo', + 'CentOS-fasttrack.repo', + 'CentOS-Media.repo', + 'CentOS-Sources.repo', + 'CentOS-Vault.repo', + 'CentOS-x86_64-kernel.repo', + 'epel.repo', + 'epel-testing.repo', + 'saltstack.repo', + 'wazuh.repo' + ] + %} + +{% elif grains.os == 'Ubuntu' %} + + {% set REPOPATH = '/etc/apt/sources.list.d/' %} + {% set ABSENTFILES = [] %} + +{% endif %} \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 6b522d03b..8a12aaa26 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -14,7 +14,6 @@ {% set CURATOR = salt['pillar.get']('curator:enabled', True) %} {% set REDIS = salt['pillar.get']('redis:enabled', True) %} {% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {% import_yaml 'salt/minion.defaults.yaml' as saltversion %} {% set saltversion = saltversion.salt.minion.version %} {% set INSTALLEDSALTVERSION = grains.saltversion %} @@ -24,18 +23,12 @@ base: 'not G@saltversion:{{saltversion}}': - match: compound - salt.minion-state-apply-test - {% if ISAIRGAP is sameas true %} - - airgap - {% endif %} + - repo.client - salt.minion 'G@os:CentOS and G@saltversion:{{saltversion}}': - match: compound - {% if ISAIRGAP is sameas true %} - - airgap - {% else %} - - yum - {% endif %} + - repo.client - yum.packages '* and G@saltversion:{{saltversion}}': diff --git a/salt/yum/init.sls b/salt/yum/init.sls deleted file mode 100644 index 339a6f2a7..000000000 --- a/salt/yum/init.sls +++ /dev/null @@ -1,17 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - -yumconf: - file.managed: - - name: /etc/yum.conf - - source: salt://yum/etc/yum.conf.jinja - - mode: 644 - - template: jinja - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} \ No newline at end of file From cf4de255eccddbef4a44c79116de9a8bd04dc567 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Apr 2021 15:18:18 -0400 Subject: [PATCH 154/258] Fix Wazuh WEL Shipping --- salt/elasticsearch/files/ingest/win.eventlogs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/win.eventlogs b/salt/elasticsearch/files/ingest/win.eventlogs index 2644be7a2..ec5cf911c 100644 --- a/salt/elasticsearch/files/ingest/win.eventlogs +++ b/salt/elasticsearch/files/ingest/win.eventlogs @@ -4,8 +4,8 @@ { "set": { "if": "ctx.winlog?.channel != null", "field": "event.module", "value": "windows_eventlog", "override": false, "ignore_failure": true } }, { "set": { "if": "ctx.winlog?.channel != null", "field": "event.dataset", "value": "{{winlog.channel}}", "override": true } }, { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, - { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, - { "set": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "value": "{{winlog.systemTime}}", "override": true } }, + { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, + { "date": { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"] } }, { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, { "set": { "field": "event.category", "value": "host", "override": true } }, { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_failure": true, "ignore_missing": true } }, From cc344d921ac481d17e5d2e6f4179215a46e1ca9e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 16:13:32 -0400 Subject: [PATCH 155/258] Skip whiptail during testing, echo error message to setup log --- setup/so-setup | 1 + setup/so-whiptail | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index bac2fcdfd..ad210048a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -575,6 +575,7 @@ if [[ $is_manager || $is_import ]]; then collect_so_allow; fi set_redirect >> $setup_log 2>&1 if [[ $is_minion ]] && ! check_manager_state; then + echo "Manager was not in a good state" >> "$setup_log" 2>&1 whiptail_manager_error fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 6ce2c214c..6f1cc6171 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1146,6 +1146,9 @@ whiptail_manager_adv_service_zeeklogs() { } whiptail_manager_error() { + + [ -n "$TESTING" ] && return + local msg read -r -d '' msg <<- EOM Setup could not determine if the manager $MSRV is in a good state. @@ -1159,9 +1162,6 @@ whiptail_manager_error() { } whiptail_manager_updates() { - - [ -n "$TESTING" ] && return - local update_string update_string=$(whiptail --title "Security Onion Setup" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ From 25637b74dbece9886e23489555ddb80b395466f5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 16:14:47 -0400 Subject: [PATCH 156/258] Add back removed testing skip --- setup/so-whiptail | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 6f1cc6171..6127a174a 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1162,6 +1162,9 @@ whiptail_manager_error() { } whiptail_manager_updates() { + + [ -n "$TESTING" ] && return + local update_string update_string=$(whiptail --title "Security Onion Setup" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ From 9d09e7bec3cbfa0d192e4df4a1594483ffbb5155 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 16:25:17 -0400 Subject: [PATCH 157/258] Fix sostatus log cron job --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index ee7cad5e6..f7a4d6731 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -316,7 +316,7 @@ sostatus_log: - mode: 644 # Install sostatus check cron -/usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: +'/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1': cron.present: - user: root - minute: '*/5' From eb7bf58f30c9bc688b799d4a3c2e1acc608bc469 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 12 Apr 2021 16:33:32 -0400 Subject: [PATCH 158/258] fix issues with repo.client state --- salt/repo/client/init.sls | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls index 60353426f..5567caac2 100644 --- a/salt/repo/client/init.sls +++ b/salt/repo/client/init.sls @@ -26,7 +26,8 @@ airgap_repo: {{ file }}: file.absent: - name: {{ REPOPATH }}{{ file }} - - onchanges_in: cleanyum + - onchanges_in: + - module: cleanyum {% endfor %} {% endif %} @@ -59,19 +60,22 @@ crsecurityonionrepo: yumconf: file.managed: - name: /etc/yum.conf - - source: salt:/repo/client/files/centos/yum.conf.jinja + - source: salt://repo/client/files/centos/yum.conf.jinja - mode: 644 - template: jinja {% endif %} cleanyum: module.run: - - pkg.clean_metadata + - pkg.clean_metadata: [] - onchanges: +{% if ISAIRGAP %} - file: airgapyum - pkgrepo: airgap_repo +{% else %} - file: crsecurityonionrepo - file: yumconf +{% endif %} {% endif %} From 325264dafd858413ff6f977e4fba8d9b20f34d0a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 12 Apr 2021 17:44:50 -0400 Subject: [PATCH 159/258] point to new repo location --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 70d453334..2a1ddab1c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -438,7 +438,7 @@ up_2.3.3X_to_2.3.50_repo() { done # Copy the new repo file if not airgap - cp $UPDATE_DIR/salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ yum clean all yum repolist fi From eb94c011e22ec056501b5aee66da88a39054a5f5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Apr 2021 11:15:15 -0400 Subject: [PATCH 160/258] update location of yum keys and repo files for setup --- salt/common/tools/sbin/so-common | 4 ++-- setup/so-functions | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index a2c28587d..97e61e6e2 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -165,9 +165,9 @@ get_random_value() { gpg_rpm_import() { if [[ "$OS" == "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then - local RPMKEYSLOC="../salt/common/keys" + local RPMKEYSLOC="../salt/repo/client/files/centos/keys" else - local RPMKEYSLOC="$UPDATEDIR/salt/common/keys" + local RPMKEYSLOC="$UPDATEDIR/salt/repo/client/files/centos/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') diff --git a/setup/so-functions b/setup/so-functions index d3a695fa1..ae65c7547 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2300,9 +2300,9 @@ securityonion_repo() { yum repolist all mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + cp -f ../salt/repo/client/files/centos/securityonioncache.repo /etc/yum.repos.d/ else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + cp -f ../salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ fi fi else From 7cbeed985a836f6a77a0904beff5c79f81652f6e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 13 Apr 2021 12:55:40 -0400 Subject: [PATCH 161/258] Differentiate between event & ingest timestamp --- salt/elasticsearch/files/ingest/osquery.query_result | 1 + salt/elasticsearch/files/ingest/win.eventlogs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result index 9bb381946..a58df1315 100644 --- a/salt/elasticsearch/files/ingest/osquery.query_result +++ b/salt/elasticsearch/files/ingest/osquery.query_result @@ -9,6 +9,7 @@ { "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } }, { "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } }, { "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.datetime", "target_field": "winlog.systemTime", "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } }, { "set": { "field": "event.module", "value": "osquery", "override": false } }, diff --git a/salt/elasticsearch/files/ingest/win.eventlogs b/salt/elasticsearch/files/ingest/win.eventlogs index ec5cf911c..a6ef87256 100644 --- a/salt/elasticsearch/files/ingest/win.eventlogs +++ b/salt/elasticsearch/files/ingest/win.eventlogs @@ -4,8 +4,8 @@ { "set": { "if": "ctx.winlog?.channel != null", "field": "event.module", "value": "windows_eventlog", "override": false, "ignore_failure": true } }, { "set": { "if": "ctx.winlog?.channel != null", "field": "event.dataset", "value": "{{winlog.channel}}", "override": true } }, { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, - { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, - { "date": { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"] } }, + { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, + { "date": { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'","yyyy-MM-dd'T'HH:mm:ss.SSSSSSS'Z'"] } }, { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, { "set": { "field": "event.category", "value": "host", "override": true } }, { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_failure": true, "ignore_missing": true } }, From 989c2b23b11540a310c86f1d11acafe197871ea4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 15:34:03 -0400 Subject: [PATCH 162/258] Fix Airgap Automation --- setup/automation/distributed-airgap-search | 1 + setup/automation/distributed-airgap-sensor | 1 + 2 files changed, 2 insertions(+) diff --git a/setup/automation/distributed-airgap-search b/setup/automation/distributed-airgap-search index 7a0888fee..1acee9b1a 100644 --- a/setup/automation/distributed-airgap-search +++ b/setup/automation/distributed-airgap-search @@ -35,6 +35,7 @@ ADMINPASS2=onionuser HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit HOSTNAME=distributed-search +INTERWEBS=AIRGAP install_type=SEARCHNODE # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= diff --git a/setup/automation/distributed-airgap-sensor b/setup/automation/distributed-airgap-sensor index 91b9c24a9..c8186bf8a 100644 --- a/setup/automation/distributed-airgap-sensor +++ b/setup/automation/distributed-airgap-sensor @@ -35,6 +35,7 @@ ZEEKVERSION=ZEEK HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit HOSTNAME=distributed-sensor +INTERWEBS=AIRGAP install_type=SENSOR # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= From 621e5c1cf819d797b77168dff12174a0fa8cc7e2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 19:18:10 -0400 Subject: [PATCH 163/258] Enable Filebeat Stats --- salt/filebeat/etc/filebeat.yml | 5 +++-- salt/filebeat/init.sls | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index c680d61c1..f6d3c5334 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -493,12 +493,13 @@ setup.template.enabled: false # append ?pretty to the URL. # Defines if the HTTP endpoint is enabled. -#http.enabled: false +http.enabled: true # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. -#http.host: localhost +http.host: localhost # Port on which the HTTP endpoint will bind. Default is 5066. +http.port: 5066 queue.mem.events: {{ FBMEMEVENTS }} queue.mem.flush.min_events: {{ FBMEMFLUSHMINEVENTS }} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 339d307ee..4d7f81819 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -74,6 +74,7 @@ so-filebeat: - port_bindings: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp + - 0.0.0.0:5066/tcp - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml From 7153f58a03cba46e5f45b334449e4944663e9350 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:17:26 -0400 Subject: [PATCH 164/258] Add Firewall for Beats port --- salt/filebeat/init.sls | 2 +- salt/firewall/portgroups.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 4d7f81819..64cdc47fc 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -74,7 +74,7 @@ so-filebeat: - port_bindings: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp - - 0.0.0.0:5066/tcp + - 0.0.0.0:5066:5066/tcp - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml diff --git a/salt/firewall/portgroups.yaml b/salt/firewall/portgroups.yaml index 55a09c6bf..1386267f5 100644 --- a/salt/firewall/portgroups.yaml +++ b/salt/firewall/portgroups.yaml @@ -18,6 +18,9 @@ firewall: beats_5644: tcp: - 5644 + beats_5066: + tcp: + - 5066 cortex: tcp: - 9001 From db7dcd76cdd969ce00ec3a6077a20407b0fdf8d2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:21:32 -0400 Subject: [PATCH 165/258] Add hostname to the listener --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index f6d3c5334..ecb16ac06 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -496,7 +496,7 @@ setup.template.enabled: false http.enabled: true # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. -http.host: localhost +http.host: {{ HOSTNAME }} # Port on which the HTTP endpoint will bind. Default is 5066. http.port: 5066 From aa66b6226fbe121c655a138346de6bce04119ef0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:22:51 -0400 Subject: [PATCH 166/258] Add hostname to the listener --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index ecb16ac06..0f7c9c778 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -496,7 +496,7 @@ setup.template.enabled: false http.enabled: true # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. -http.host: {{ HOSTNAME }} +http.host: 0.0.0.0 # Port on which the HTTP endpoint will bind. Default is 5066. http.port: 5066 From 904d34977f5508c5debf3b2ab6e306714a799e2c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:48:53 -0400 Subject: [PATCH 167/258] Add telegraf scripts to track eps and failures for beats --- salt/telegraf/etc/telegraf.conf | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 1b172485b..03113a3f4 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -679,7 +679,8 @@ "/scripts/redis.sh", "/scripts/influxdbsize.sh", "/scripts/eps.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" ## Timeout for each command to complete. @@ -695,7 +696,8 @@ "/scripts/zeekcaptureloss.sh", {% endif %} "/scripts/oldpcap.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -711,7 +713,8 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/eps.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -729,7 +732,8 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/eps.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -746,7 +750,8 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/influxdbsize.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" From 6fc7ed1a25b437bde17774bbdf309ab4bc3068b4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:51:27 -0400 Subject: [PATCH 168/258] Add telegraf scripts to track eps and failures for beats --- salt/telegraf/scripts/beatseps.sh | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 salt/telegraf/scripts/beatseps.sh diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh new file mode 100644 index 000000000..faba0fabc --- /dev/null +++ b/salt/telegraf/scripts/beatseps.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=beatseps +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +PREVCOUNTFILE='/tmp/beatseps.txt' +EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" +FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" + +if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi + + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + +fi + +exit 0 From a50b3e84755369484d2b5e18c6e672e3898c5d9e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 10:22:06 -0400 Subject: [PATCH 169/258] add delay to salt-minion service starting - https://github.com/Security-Onion-Solutions/securityonion/issues/3543 --- salt/salt/minion.defaults.yaml | 3 ++- salt/salt/minion.sls | 17 +++++++++++- salt/salt/service/salt-minion.service.jinja | 30 +++++++++++++++++++++ salt/systemd/reload.sls | 3 +++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 salt/salt/service/salt-minion.service.jinja create mode 100644 salt/systemd/reload.sls diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 9d888f106..560493bed 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -3,4 +3,5 @@ salt: minion: version: 3003 - check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default \ No newline at end of file + check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default + service_start_delay: 30 # in seconds. \ No newline at end of file diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 004fddebe..e4e339b42 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,9 +2,11 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} +{% from 'salt/minion.defaults.yaml' import salt.minion.service_start_delay as service_start_delay %} include: - salt + - systemd.reload install_salt_minion: cmd.run: @@ -32,8 +34,21 @@ set_log_levels: - listen_in: - service: salt_minion_service +salt_minion_service_unit_file: + file.managed: + - name: /etc/systemd/system/multi-user.target.wants/salt-minion.service + - source: salt://salt/service/salt-minion.servic.jinja + - template: jinja + - defaults: + - service_start_delay: {{ service_start_delay }} + - onchanges_in: + - module: systemd_reload + - listen_in: + - service: salt_minion_service + salt_minion_service: service.running: - name: salt-minion - enable: True - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" \ No newline at end of file + - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" + diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja new file mode 100644 index 000000000..aea68b994 --- /dev/null +++ b/salt/salt/service/salt-minion.service.jinja @@ -0,0 +1,30 @@ +[Unit] +Description=The Salt Minion +Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html +After=network.target salt-master.service + +[Service] +KillMode=process +Type=notify +NotifyAccess=all +LimitNOFILE=8192 +ExecStart=/usr/bin/salt-minion +ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} + +[Install] +WantedBy=multi-user.target + + + + + + + + + + + + + + + diff --git a/salt/systemd/reload.sls b/salt/systemd/reload.sls new file mode 100644 index 000000000..ff2185539 --- /dev/null +++ b/salt/systemd/reload.sls @@ -0,0 +1,3 @@ +systemd_reload: + module.run: + - service.systemctl_reload: [] \ No newline at end of file From 9bfdae9cd5c9bb8f97694a57db80374a6d32e3f5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:06:06 -0400 Subject: [PATCH 170/258] fix import --- salt/salt/minion.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index e4e339b42..a2b8a17e3 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,7 +2,8 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} -{% from 'salt/minion.defaults.yaml' import salt.minion.service_start_delay as service_start_delay %} +{% from 'salt/minion.defaults.yaml' import salt as SALTMINION %} +{% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} include: - salt From 47aa66876d27c4d99cf12da999ea934542918e60 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:07:16 -0400 Subject: [PATCH 171/258] fix import --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index a2b8a17e3..2c023cbdd 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,7 +2,7 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} -{% from 'salt/minion.defaults.yaml' import salt as SALTMINION %} +{% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} {% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} include: From 48c531bc2c6de762e2859ff7e65bd8a977aa3266 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:09:13 -0400 Subject: [PATCH 172/258] fix file defaults def --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 2c023cbdd..9d16846d9 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -41,7 +41,7 @@ salt_minion_service_unit_file: - source: salt://salt/service/salt-minion.servic.jinja - template: jinja - defaults: - - service_start_delay: {{ service_start_delay }} + service_start_delay: {{ service_start_delay }} - onchanges_in: - module: systemd_reload - listen_in: From d003d4941b56e100a8eae657d206df68ee3a5a17 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:10:19 -0400 Subject: [PATCH 173/258] fix bad typing --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 9d16846d9..44f866880 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -38,7 +38,7 @@ set_log_levels: salt_minion_service_unit_file: file.managed: - name: /etc/systemd/system/multi-user.target.wants/salt-minion.service - - source: salt://salt/service/salt-minion.servic.jinja + - source: salt://salt/service/salt-minion.service.jinja - template: jinja - defaults: service_start_delay: {{ service_start_delay }} From f60da54ff030c2db9da65f87f4cbeb4a55c67c75 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:11:13 -0400 Subject: [PATCH 174/258] remove extra lines at end --- salt/salt/service/salt-minion.service.jinja | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja index aea68b994..c7bae0bc2 100644 --- a/salt/salt/service/salt-minion.service.jinja +++ b/salt/salt/service/salt-minion.service.jinja @@ -12,19 +12,4 @@ ExecStart=/usr/bin/salt-minion ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} [Install] -WantedBy=multi-user.target - - - - - - - - - - - - - - - +WantedBy=multi-user.target \ No newline at end of file From 71d7ca8958bdd68e01a98949049910a8577ca734 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 15:48:33 -0400 Subject: [PATCH 175/258] only manage service file if the right salt version is installed --- salt/salt/minion.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 44f866880..1c7f1a5e8 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -46,6 +46,7 @@ salt_minion_service_unit_file: - module: systemd_reload - listen_in: - service: salt_minion_service + - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" salt_minion_service: service.running: From f61bf73f972eafce84fd71fb92c73323c547b153 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 15 Apr 2021 11:59:34 +0000 Subject: [PATCH 176/258] Remove ESUSER/ESPASS for now to prevent issues with attempting Elastic Auth when connecting to ES. --- salt/soctopus/files/SOCtopus.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index c9941c3e1..b6ee45e74 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -8,8 +8,8 @@ [es] es_url = https://{{MANAGER}}:9200 es_ip = {{MANAGER}} -es_user = YOURESUSER -es_pass = YOURESPASS +es_user = +es_pass = es_index_pattern = so-* es_verifycert = no From 2f198ed9fb8b5646e469432fbdf6dac47d17abea Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 09:42:00 -0400 Subject: [PATCH 177/258] change how salt is held and unheld from updates --- salt/salt/map.jinja | 14 +++----------- salt/salt/minion.sls | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 6b5273b84..67742812b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -1,5 +1,6 @@ {% import_yaml 'salt/minion.defaults.yaml' as saltminion %} {% set SALTVERSION = saltminion.salt.minion.version %} +{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} @@ -7,20 +8,11 @@ {% set SPLITCHAR = '-' %} {% endif %} -{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} - -{% if grains.os|lower == 'ubuntu' %} - {% set COMMON = 'salt-common' %} -{% elif grains.os|lower in ['centos', 'redhat'] %} - {% set COMMON = 'salt' %} -{% endif %} - {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} - {% set UPGRADECOMMAND = 'yum clean all ; yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common ; apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; apt-mark hold salt-common && apt-mark hold salt-minion' %} + {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 1c7f1a5e8..6488124f6 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -9,6 +9,12 @@ include: - salt - systemd.reload +{% if "{{INSTALLEDSALTVERSION}}" != "{{SALTVERSION}}" %} +unhold_salt_packages: + module.run: + - pkg.unhold: + - 'salt-*' + install_salt_minion: cmd.run: - name: | @@ -16,15 +22,13 @@ install_salt_minion: exec 1>&- # close stdout exec 2>&- # close stderr nohup /bin/sh -c '{{ UPGRADECOMMAND }}' & - - onlyif: test "{{INSTALLEDSALTVERSION}}" != "{{SALTVERSION}}" +{% endif %} -salt_minion_package: - pkg.installed: - - pkgs: - - {{ COMMON }} - - salt-minion - - hold: True - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" +{% if "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" %} +hold_salt_packages: + module.run: + - pkg.hold: + - 'salt-*' set_log_levels: file.append: @@ -46,11 +50,9 @@ salt_minion_service_unit_file: - module: systemd_reload - listen_in: - service: salt_minion_service - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" salt_minion_service: service.running: - name: salt-minion - enable: True - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" - +{% endif %} From 22edbcc1112cbec4f93a525443c8de292448336e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 11:29:01 -0400 Subject: [PATCH 178/258] can use SPLITCHAR before defined --- salt/salt/map.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 67742812b..3ba7194f5 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -1,6 +1,5 @@ {% import_yaml 'salt/minion.defaults.yaml' as saltminion %} {% set SALTVERSION = saltminion.salt.minion.version %} -{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} @@ -8,6 +7,8 @@ {% set SPLITCHAR = '-' %} {% endif %} +{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} + {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} From 9d01387a04725611b3c7b96cafbd9bebe48070c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 11:57:25 -0400 Subject: [PATCH 179/258] remove references to the common salt package --- salt/salt/master.sls | 12 ++++-------- salt/salt/minion.sls | 1 - 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 3c23bbb36..d0a655051 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -1,17 +1,13 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% from 'salt/map.jinja' import COMMON with context %} - include: - salt.minion -salt_master_package: - pkg.installed: - - pkgs: - - {{ COMMON }} - - salt-master - - hold: True +hold_salt_master_package: + module.run: + - pkg.hold: + - 'salt-master' salt_master_service: service.running: diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 6488124f6..e656ae8a6 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -1,4 +1,3 @@ -{% from 'salt/map.jinja' import COMMON with context %} {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} From 9d676efada5b07a38d2140d387b6d1b4072b1a2b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 12:45:34 -0400 Subject: [PATCH 180/258] move salt_minion_service state outside jinja if --- salt/salt/minion.sls | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index e656ae8a6..1b0f7d901 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -49,9 +49,11 @@ salt_minion_service_unit_file: - module: systemd_reload - listen_in: - service: salt_minion_service +{% endif %} +# this has to be outside the if statement above since there are _in calls to this state salt_minion_service: service.running: - name: salt-minion - enable: True -{% endif %} + - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" \ No newline at end of file From 24b263c81227052bceaa468ac2175e7683ddeb1d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 16 Apr 2021 11:37:18 -0400 Subject: [PATCH 181/258] only hold/unhold packages if not already unheld/held --- salt/salt/map.jinja | 2 ++ salt/salt/minion.sls | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 3ba7194f5..5c1689e6c 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -3,8 +3,10 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} + {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?') %} {% else %} {% set SPLITCHAR = '-' %} + {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?') %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 1b0f7d901..5145da34b 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -1,6 +1,7 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} +{% from 'salt/map.jinja' import SALTNOTHELD %} {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} {% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} @@ -8,11 +9,14 @@ include: - salt - systemd.reload -{% if "{{INSTALLEDSALTVERSION}}" != "{{SALTVERSION}}" %} +{% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} + +{% if SALTNOTHELD == 0 %} unhold_salt_packages: module.run: - pkg.unhold: - - 'salt-*' + - name: 'salt-*' +{% endif %} install_salt_minion: cmd.run: @@ -23,11 +27,14 @@ install_salt_minion: nohup /bin/sh -c '{{ UPGRADECOMMAND }}' & {% endif %} -{% if "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" %} +{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} + +{% if SALTNOTHELD == 1 %} hold_salt_packages: module.run: - pkg.hold: - - 'salt-*' + - name: 'salt-*' +{% endif %} set_log_levels: file.append: From 1b15f018742d7f28bcce7b76582ec189fd9c3aea Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 16 Apr 2021 13:09:01 -0400 Subject: [PATCH 182/258] fix salt.master state --- salt/salt/master.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/salt/master.sls b/salt/salt/master.sls index d0a655051..8b2b6c7d0 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -1,13 +1,16 @@ +{% from 'salt/map.jinja' import SALTNOTHELD %} {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} include: - salt.minion +{% if SALTNOTHELD == 1 %} hold_salt_master_package: module.run: - pkg.hold: - - 'salt-master' + - name: salt-master +{% endif %} salt_master_service: service.running: From 58febe795574fd200bcb5e48c77659c75c8fd4e5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 16 Apr 2021 16:04:07 -0400 Subject: [PATCH 183/258] [fix] so-docker-prune breaks when multiple "so-" images share a version --- salt/common/tools/sbin/so-docker-prune | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 5a56f506d..f6c043ef3 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -60,15 +60,19 @@ def main(quiet): no_prunable = True for t_list in grouped_tag_lists: try: - # Keep the 2 most current images + # Group tags by version, in case multiple images exist with the same version string t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True) - if len(t_list) <= 2: + grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ] + + # Keep the 2 most current version groups + if len(grouped_t_list) <= 2: continue else: no_prunable = False - for tag in t_list[2:]: - if not quiet: print(f'Removing image {tag}') - client.images.remove(tag) + for group in grouped_t_list[2:]: + for tag in group: + if not quiet: print(f'Removing image {tag}') + client.images.remove(tag) except InvalidVersion as e: print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr) exit(1) From ef0669aabb8c0fb685ce2742a0ec1545891fbfb8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sat, 17 Apr 2021 18:24:33 -0400 Subject: [PATCH 184/258] Fix beat script location --- salt/telegraf/etc/telegraf.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 03113a3f4..0798fc920 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -680,7 +680,7 @@ "/scripts/influxdbsize.sh", "/scripts/eps.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" ## Timeout for each command to complete. @@ -697,7 +697,7 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -714,7 +714,7 @@ "/scripts/oldpcap.sh", "/scripts/eps.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -733,7 +733,7 @@ "/scripts/oldpcap.sh", "/scripts/eps.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -751,7 +751,7 @@ "/scripts/oldpcap.sh", "/scripts/influxdbsize.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" From 9e57fd2df0b88f18bc02629fe185dc47c0716169 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 19 Apr 2021 09:00:30 -0400 Subject: [PATCH 185/258] cant pipe to grep without , python_shell=True --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5c1689e6c..5d6d980be 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -3,10 +3,10 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} - {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?') %} + {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} {% else %} {% set SPLITCHAR = '-' %} - {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?') %} + {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} From e8553162a53fa61b55673aaa306ef28cfa09167c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 10:50:42 -0400 Subject: [PATCH 186/258] [refactor] Change how whiptail asks for proxy settings --- salt/manager/init.sls | 6 -- setup/automation/distributed-airgap-manager | 1 - setup/automation/distributed-ami-manager | 1 - setup/automation/distributed-iso-manager | 1 - .../automation/distributed-net-centos-manager | 1 - .../automation/distributed-net-ubuntu-manager | 1 - .../distributed-net-ubuntu-suricata-manager | 1 - setup/automation/eval-airgap | 1 - setup/automation/eval-ami | 1 - setup/automation/eval-iso | 1 - setup/automation/eval-net-centos | 1 - setup/automation/eval-net-ubuntu | 1 - setup/automation/import-airgap | 1 - setup/automation/import-ami | 1 - setup/automation/import-iso | 1 - setup/automation/import-net-centos | 1 - setup/automation/import-net-ubuntu | 1 - setup/automation/standalone-airgap | 1 - setup/automation/standalone-ami | 1 - setup/automation/standalone-iso | 1 - setup/automation/standalone-iso-suricata | 1 - setup/automation/standalone-net-centos | 1 - setup/automation/standalone-net-centos-proxy | 1 - setup/automation/standalone-net-ubuntu | 1 - setup/so-functions | 27 ++++-- setup/so-setup | 11 +-- setup/so-whiptail | 89 ++++++++++++++----- 27 files changed, 89 insertions(+), 67 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 908ef4502..1d21c95d3 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -18,7 +18,6 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set managerproxy = salt['pillar.get']('global:managerupdate', '0') %} {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %} socore_own_saltstack: @@ -35,8 +34,6 @@ socore_own_saltstack: - mode: 750 - replace: False -{% if managerproxy == 1 %} - # Create the directories for apt-cacher-ng aptcacherconfdir: file.directory: @@ -60,7 +57,6 @@ aptcacherlogdir: - makedirs: true # Copy the config - acngcopyconf: file.managed: - name: /opt/so/conf/aptcacher-ng/etc/acng.conf @@ -84,8 +80,6 @@ append_so-aptcacherng_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-aptcacherng -{% endif %} - strelka_yara_update_old_1: cron.absent: - user: root diff --git a/setup/automation/distributed-airgap-manager b/setup/automation/distributed-airgap-manager index 3ed1a34f8..ddf44c100 100644 --- a/setup/automation/distributed-airgap-manager +++ b/setup/automation/distributed-airgap-manager @@ -42,7 +42,6 @@ INTERWEBS=AIRGAP # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/distributed-ami-manager b/setup/automation/distributed-ami-manager index 793e07ceb..6f5fb93dc 100644 --- a/setup/automation/distributed-ami-manager +++ b/setup/automation/distributed-ami-manager @@ -41,7 +41,6 @@ install_type=MANAGER # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/distributed-iso-manager b/setup/automation/distributed-iso-manager index 72cedb75e..07a22b588 100644 --- a/setup/automation/distributed-iso-manager +++ b/setup/automation/distributed-iso-manager @@ -41,7 +41,6 @@ install_type=MANAGER # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/distributed-net-centos-manager b/setup/automation/distributed-net-centos-manager index 72cedb75e..07a22b588 100644 --- a/setup/automation/distributed-net-centos-manager +++ b/setup/automation/distributed-net-centos-manager @@ -41,7 +41,6 @@ install_type=MANAGER # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/distributed-net-ubuntu-manager b/setup/automation/distributed-net-ubuntu-manager index 104bf4df4..712db3020 100644 --- a/setup/automation/distributed-net-ubuntu-manager +++ b/setup/automation/distributed-net-ubuntu-manager @@ -41,7 +41,6 @@ install_type=MANAGER # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/distributed-net-ubuntu-suricata-manager b/setup/automation/distributed-net-ubuntu-suricata-manager index d1fdf158d..30aebc122 100644 --- a/setup/automation/distributed-net-ubuntu-suricata-manager +++ b/setup/automation/distributed-net-ubuntu-suricata-manager @@ -41,7 +41,6 @@ install_type=MANAGER # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/eval-airgap b/setup/automation/eval-airgap index 095075a6b..e8deebe69 100644 --- a/setup/automation/eval-airgap +++ b/setup/automation/eval-airgap @@ -42,7 +42,6 @@ INTERWEBS=AIRGAP # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/eval-ami b/setup/automation/eval-ami index 1efab191d..ac8e42728 100644 --- a/setup/automation/eval-ami +++ b/setup/automation/eval-ami @@ -41,7 +41,6 @@ install_type=EVAL # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/eval-iso b/setup/automation/eval-iso index 880b3cc0c..d8a8c800a 100644 --- a/setup/automation/eval-iso +++ b/setup/automation/eval-iso @@ -41,7 +41,6 @@ install_type=EVAL # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/eval-net-centos b/setup/automation/eval-net-centos index 82d2cc9ec..5c0ea36a3 100644 --- a/setup/automation/eval-net-centos +++ b/setup/automation/eval-net-centos @@ -41,7 +41,6 @@ install_type=EVAL # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/eval-net-ubuntu b/setup/automation/eval-net-ubuntu index 132b8766e..4dc0eceda 100644 --- a/setup/automation/eval-net-ubuntu +++ b/setup/automation/eval-net-ubuntu @@ -41,7 +41,6 @@ install_type=EVAL # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-airgap b/setup/automation/import-airgap index 9c394ef2f..dc524e0c3 100644 --- a/setup/automation/import-airgap +++ b/setup/automation/import-airgap @@ -42,7 +42,6 @@ INTERWEBS=AIRGAP # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-ami b/setup/automation/import-ami index 10758be9a..039e9caee 100644 --- a/setup/automation/import-ami +++ b/setup/automation/import-ami @@ -41,7 +41,6 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-iso b/setup/automation/import-iso index fbfdd364b..6cc3106fd 100644 --- a/setup/automation/import-iso +++ b/setup/automation/import-iso @@ -41,7 +41,6 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-net-centos b/setup/automation/import-net-centos index f6394bde1..2536c8516 100644 --- a/setup/automation/import-net-centos +++ b/setup/automation/import-net-centos @@ -41,7 +41,6 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-net-ubuntu b/setup/automation/import-net-ubuntu index ded17d09f..dc72c8184 100644 --- a/setup/automation/import-net-ubuntu +++ b/setup/automation/import-net-ubuntu @@ -41,7 +41,6 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/standalone-airgap b/setup/automation/standalone-airgap index 649b51e3c..99b003e05 100644 --- a/setup/automation/standalone-airgap +++ b/setup/automation/standalone-airgap @@ -42,7 +42,6 @@ INTERWEBS=AIRGAP # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/standalone-ami b/setup/automation/standalone-ami index 7200d3637..c006b28fb 100644 --- a/setup/automation/standalone-ami +++ b/setup/automation/standalone-ami @@ -41,7 +41,6 @@ install_type=STANDALONE # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/standalone-iso b/setup/automation/standalone-iso index dd0edb67f..ec972b066 100644 --- a/setup/automation/standalone-iso +++ b/setup/automation/standalone-iso @@ -41,7 +41,6 @@ install_type=STANDALONE # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/standalone-iso-suricata b/setup/automation/standalone-iso-suricata index f4697f308..d6dbc73d2 100644 --- a/setup/automation/standalone-iso-suricata +++ b/setup/automation/standalone-iso-suricata @@ -41,7 +41,6 @@ install_type=STANDALONE # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/standalone-net-centos b/setup/automation/standalone-net-centos index 6b7a7ebac..a711ba878 100644 --- a/setup/automation/standalone-net-centos +++ b/setup/automation/standalone-net-centos @@ -41,7 +41,6 @@ install_type=STANDALONE # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/standalone-net-centos-proxy b/setup/automation/standalone-net-centos-proxy index ee2504a98..1fc245cba 100644 --- a/setup/automation/standalone-net-centos-proxy +++ b/setup/automation/standalone-net-centos-proxy @@ -41,7 +41,6 @@ install_type=STANDALONE # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/standalone-net-ubuntu b/setup/automation/standalone-net-ubuntu index fafb98cd4..a30e2a444 100644 --- a/setup/automation/standalone-net-ubuntu +++ b/setup/automation/standalone-net-ubuntu @@ -41,7 +41,6 @@ install_type=STANDALONE # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/so-functions b/setup/so-functions index a37867b5a..7dd5511fb 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -480,6 +480,21 @@ collect_mtu() { done } +collect_net_method() { + whiptail_net_method + + [[ -n $network_traffic ]] && collect_proxy + + if [[ "$network_traffic" == *"_MANAGER" ]]; then + whiptail_manager_updates_warning + MANAGERUPDATES=1 + fi + + if [[ "$network_traffic" == "PROXY"* ]]; then + collect_proxy no_ask + fi +} + collect_node_es_heap() { whiptail_node_es_heap "$ES_HEAP_SIZE" } @@ -582,7 +597,9 @@ collect_patch_schedule_name_import() { collect_proxy() { [[ -n $TESTING ]] && return - collect_proxy_details || return + local ask=${1:-true} + + collect_proxy_details "$ask" || return while ! proxy_validate; do if whiptail_invalid_proxy; then collect_proxy_details no_ask @@ -2671,10 +2688,10 @@ set_redirect() { set_updates() { if [ "$MANAGERUPDATES" = '1' ]; then if [ "$OS" = 'centos' ]; then - if [[ ! $is_airgap ]]; then - if ! grep -q "$MSRV" /etc/yum.conf; then - echo "proxy=http://$MSRV:3142" >> /etc/yum.conf - fi + if [[ ! $is_airgap ]]; then + if ! grep -q "$MSRV" /etc/yum.conf; then + echo "proxy=http://$MSRV:3142" >> /etc/yum.conf + fi fi else # Set it up so the updates roll through the manager diff --git a/setup/so-setup b/setup/so-setup index ad210048a..0667c99db 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -211,7 +211,7 @@ if ! [[ -f $install_opt_file ]]; then set_main_ip >> $setup_log 2>&1 compare_main_nic_ip reset_proxy - collect_proxy + collect_net_method [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 whiptail_net_setup_complete else @@ -319,7 +319,7 @@ if ! [[ -f $install_opt_file ]]; then reset_proxy if [[ -z $is_airgap ]]; then - collect_proxy + collect_net_method [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi @@ -499,13 +499,6 @@ if [[ $is_manager || $is_import ]]; then get_redirect fi -if [[ ! $is_airgap && ( $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ) ]]; then - whiptail_manager_updates - if [[ $setup_type == 'network' && $MANAGERUPDATES == 1 ]]; then - whiptail_manager_updates_warning - fi -fi - if [[ $is_distmanager ]]; then collect_soremote_inputs fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 6127a174a..06a1afec1 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1027,6 +1027,68 @@ whiptail_management_interface_setup() { whiptail_check_exitstatus $exitstatus } +whiptail_net_method() { + [ -n "$TESTING" ] && return + + [[ $is_airgap ]] && return + + local pkg_mngr + if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi + + read -r -d '' options_msg <<- EOM + "Direct" - Internet requests connect directly to the Internet. + + EOM + local options=( + " Direct" "" + ) + local proxy_desc="proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment." + + if [[ $is_minion ]]; then + local mngr_article + if [[ $is_distmanager ]]; then mngr_article="this"; else mngr_article="the"; fi + + read -r -d '' options_msg <<- EOM + ${options_msg} + + "Direct + Manager" - all traffic passes to the Internet normally, but ${pkg_mngr} updates will instead be pulled from ${mngr_article} manager. + + "Proxy" - ${proxy_desc} + + "Proxy + Manager" - proxy all traffic from the "Proxy" option except ${pkg_mngr} updates, which will instead pull from the manager. + EOM + + options+=( + " Direct + Manager" "" + " Proxy" "" + " Proxy + Manager" "" + ) + local height=25 + else + read -r -d '' options_msg <<- EOM + ${options_msg} + + "Proxy" - ${proxy_desc} + EOM + options+=( + " Proxy" "" + ) + local height=17 + fi + + local msg + read -r -d '' msg <<- EOM + How would you like to connect to the Internet? + + $options_msg + EOM + + local option_count=$(( ${#options[@]} / 2 )) + + network_traffic=$(whiptail --title "Security Onion Setup" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) + network_traffic=$(echo "${network_traffic^^}" | tr -d ' ' | tr '+' '_') +} + whiptail_net_setup_complete() { [ -n "$TESTING" ] && return @@ -1161,29 +1223,6 @@ whiptail_manager_error() { whiptail --title "Security Onion Setup" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 } -whiptail_manager_updates() { - - [ -n "$TESTING" ] && return - - local update_string - update_string=$(whiptail --title "Security Onion Setup" --radiolist \ - "How would you like to download OS package updates for your grid?" 20 75 4 \ - "MANAGER" "Manager node is proxy for updates" ON \ - "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - - case "$update_string" in - 'MANAGER') - export MANAGERUPDATES='1' - ;; - *) - export MANAGERUPDATES='0' - ;; - esac - -} - whiptail_manager_updates_warning() { [ -n "$TESTING" ] && return @@ -1485,7 +1524,9 @@ whiptail_patch_schedule_select_hours() { whiptail_proxy_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Do you want to set a proxy server for this installation?" 7 60 --defaultno + local pkg_mngr + if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi + whiptail --title "Security Onion Setup" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno } whiptail_proxy_addr() { From c907d416dfd623601dc27b80625d774f8907ad5c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 11:27:17 -0400 Subject: [PATCH 187/258] Set proxy for apt cacher too --- salt/manager/files/acng/acng.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index 3492cf111..df934643b 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -90,3 +90,7 @@ PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirr # MaxDlSpeed: 500 # MaxInresponsiveDlSize: 64000 # BadRedirDetectMime: text/html +{% set proxy = salt['pillar.get']('manager:proxy') -%} +{{ if proxy }} +Proxy: {{ proxy }} +{{ endif }} From 31f193c397c26cc2cdf7d088f1c30795048d602a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 12:36:46 -0400 Subject: [PATCH 188/258] Change EPS for Telegraf --- salt/telegraf/etc/telegraf.conf | 41 ++++++++++++++++-- salt/telegraf/scripts/beatseps.sh | 50 ++++++++++------------ salt/telegraf/scripts/checkfiles.sh | 18 ++++---- salt/telegraf/scripts/eps.sh | 50 ++++++++++------------ salt/telegraf/scripts/helixeps.sh | 48 ++++++++++----------- salt/telegraf/scripts/influxdbsize.sh | 18 ++++---- salt/telegraf/scripts/oldpcap.sh | 24 +++++------ salt/telegraf/scripts/raid.sh | 20 ++++----- salt/telegraf/scripts/redis.sh | 20 ++++----- salt/telegraf/scripts/sostatus.sh | 24 +++++------ salt/telegraf/scripts/stenoloss.sh | 50 +++++++++++----------- salt/telegraf/scripts/suriloss.sh | 49 ++++++++++------------ salt/telegraf/scripts/zeekcaptureloss.sh | 53 +++++++++++------------- salt/telegraf/scripts/zeekloss.sh | 53 +++++++++++------------- 14 files changed, 255 insertions(+), 263 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0798fc920..dac9bf60e 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -17,6 +17,7 @@ {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} +{%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} # Global tags can be specified here in key="value" format. [global_tags] @@ -614,18 +615,29 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false +{% if TRUE_CLUSTER %} + {% if grains.role == 'so-manager' %} +[[inputs.elasticsearch]] + servers = ["https://{{ MANAGER }}:9200"] + insecure_skip_verify = true + local = false + cluster_health = true + cluster_stats = true + {% endif %} + +{% else %} # # Read stats from one or more Elasticsearch servers or clusters -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} + {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] servers = ["https://{{ MANAGER }}:9200"] insecure_skip_verify = true -{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} + {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] insecure_skip_verify = true + {% endif %} {% endif %} - # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" @@ -673,11 +685,32 @@ # ## Commands array -{% if grains['role'] in ['so-manager', 'so-managersearch'] %} +{% if grains['role'] in ['so-manager'] %} [[inputs.exec]] commands = [ "/scripts/redis.sh", "/scripts/influxdbsize.sh", + "/scripts/raid.sh", + "/scripts/beatseps.sh" + ] + data_format = "influx" + ## Timeout for each command to complete. + timeout = "15s" +{% elif grains['role'] in ['so-managersearch'] %} +[[inputs.exec]] + commands = [ + "/scripts/redis.sh", + "/scripts/influxdbsize.sh", + "/scripts/eps.sh", + "/scripts/raid.sh", + "/scripts/beatseps.sh" + ] + data_format = "influx" + ## Timeout for each command to complete. + timeout = "15s" +{% elif grains['role'] in ['so-node'] %} +[[inputs.exec]] + commands = [ "/scripts/eps.sh", "/scripts/raid.sh", "/scripts/beatseps.sh" diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index faba0fabc..1226c42e4 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -15,37 +15,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=beatseps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/beatseps.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" -FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/beatseps.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" + FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" + + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" - fi -exit 0 diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index c84b6bec9..12cf3ece6 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=checkfiles -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) +if [ ! $THEGREP ]; then -echo "faffiles files=$FILES" + FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) + + echo "faffiles files=$FILES" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index dcc4b9051..99f001552 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -15,36 +15,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=eps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/eps.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/eps.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi + + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "consumptioneps eps=${EVENTS%%.*}" + +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "esteps eps=${EVENTS%%.*}" - fi -exit 0 diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index be5aaa1d2..7922a7ab6 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -15,35 +15,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=helixeps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/helixevents.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/helixevents.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" + + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "helixeps eps=${EVENTS%%.*}" +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "helixeps eps=${EVENTS%%.*}" - -fi - -exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/influxdbsize.sh b/salt/telegraf/scripts/influxdbsize.sh index 9bab7815b..f2ed41f35 100644 --- a/salt/telegraf/scripts/influxdbsize.sh +++ b/salt/telegraf/scripts/influxdbsize.sh @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=influxsize -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) +if [ ! $THEGREP ]; then -echo "influxsize kbytes=$INFLUXSIZE" + INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) + + echo "influxsize kbytes=$INFLUXSIZE" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index 0557137e7..d43f16d14 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,18 +15,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=oldpcap -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -# Get the data -OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) -DATE=$(date +%s) -AGE=$(($DATE - $OLDPCAP)) +if [ ! $THEGREP ]; then -echo "pcapage seconds=$AGE" + # Get the data + OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + DATE=$(date +%s) + AGE=$(($DATE - $OLDPCAP)) + + echo "pcapage seconds=$AGE" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index 0938bb658..c0aabe75f 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -15,19 +15,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=raid -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf -RAIDLOG=/var/log/raid/status.log -RAIDSTATUS=$(cat /var/log/raid/status.log) +THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ -f "$RAIDLOG" ]; then - echo "raid $RAIDSTATUS" +if [ ! $THEGREP ]; then + + if [ -f "$RAIDLOG" ]; then + echo "raid $RAIDSTATUS" + else + exit 0 + fi else exit 0 fi diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index 04079c63b..613e6cdea 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -16,16 +16,14 @@ # along with this program. If not, see . -APP=redis -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') -PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') +if [ ! $THEGREP ]; then -echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED" + UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') + PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') + + echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED" +else + exit 0 +fi diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh index 23096d903..1baf38d2b 100644 --- a/salt/telegraf/scripts/sostatus.sh +++ b/salt/telegraf/scripts/sostatus.sh @@ -14,20 +14,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +THEGREP=$(ps -ef | grep $0 | grep -v grep) -APP=sostatus -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf -SOSTATUSLOG=/var/log/sostatus/status.log -SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) +if [ ! $THEGREP ]; then -if [ -f "$SOSTATUSLOG" ]; then - echo "sostatus status=$SOSTATUSSTATUS" -else + SOSTATUSLOG=/var/log/sostatus/status.log + SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) + + if [ -f "$SOSTATUSLOG" ]; then + echo "sostatus status=$SOSTATUSSTATUS" + else + exit 0 + fi +else exit 0 fi diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index ad88ccc8d..a5c974a73 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -15,31 +15,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=stenoloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -TSFILE=/var/log/telegraf/laststenodrop.log -if [ -f "$TSFILE" ]; then - LASTTS=$(cat $TSFILE) +if [ ! $THEGREP ]; then + + TSFILE=/var/log/telegraf/laststenodrop.log + if [ -f "$TSFILE" ]; then + LASTTS=$(cat $TSFILE) + else + LASTTS=0 + fi + + # Get the data + LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) + CURRENTTS=$(echo $LOGLINE | awk '{print $1}') + + if [[ "$CURRENTTS" != "$LASTTS" ]]; then + DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') + echo $CURRENTTS > $TSFILE + else + DROP=0 + fi + + echo "stenodrop drop=$DROP" else - LASTTS=0 -fi - -# Get the data -LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) -CURRENTTS=$(echo $LOGLINE | awk '{print $1}') - -if [[ "$CURRENTTS" != "$LASTTS" ]]; then - DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') - echo $CURRENTTS > $TSFILE -else - DROP=0 -fi - -echo "stenodrop drop=$DROP" \ No newline at end of file + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 08f8c23eb..0ae867b29 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -16,37 +16,32 @@ # along with this program. If not, see . -APP=suriloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) -CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) +if [ ! $THEGREP ]; then -if [ $CHECKIT == 2 ]; then - declare RESULT=($SURILOG) + SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) + CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) - CURRENTDROP=${RESULT[4]} - PASTDROP=${RESULT[14]} - DROPPED=$((CURRENTDROP - PASTDROP)) - if [ $DROPPED == 0 ]; then - LOSS=0 - echo "suridrop drop=0" - else - CURRENTPACKETS=${RESULT[9]} - PASTPACKETS=${RESULT[19]} - TOTALCURRENT=$((CURRENTPACKETS + CURRENTDROP)) - TOTALPAST=$((PASTPACKETS + PASTDROP)) - TOTAL=$((TOTALCURRENT - TOTALPAST)) + if [ $CHECKIT == 2 ]; then + declare RESULT=($SURILOG) - LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) - echo "suridrop drop=$LOSS" - fi + CURRENTDROP=${RESULT[4]} + PASTDROP=${RESULT[14]} + DROPPED=$((CURRENTDROP - PASTDROP)) + if [ $DROPPED == 0 ]; then + LOSS=0 + echo "suridrop drop=0" + else + CURRENTPACKETS=${RESULT[9]} + PASTPACKETS=${RESULT[19]} + TOTALCURRENT=$((CURRENTPACKETS + CURRENTDROP)) + TOTALPAST=$((PASTPACKETS + PASTDROP)) + TOTAL=$((TOTALCURRENT - TOTALPAST)) + + LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) + echo "suridrop drop=$LOSS" + fi else echo "suridrop drop=0" fi \ No newline at end of file diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index aa8a222a3..995971b18 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -18,35 +18,32 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp -APP=zeekcaploss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} - ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log -elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then - WORKERS=1 - ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log -else - echo 'Zeek capture_loss.log not found' >/dev/stderr - exit 2 -fi +if [ ! $THEGREP ]; then -LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt -if [ -f "$ZEEKLOG" ]; then - CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g') - if [ -f "$LASTCAPTURELOSSLOG" ]; then - LASTTS=$(cat $LASTCAPTURELOSSLOG) - if [[ "$LASTTS" != "$CURRENTTS" ]]; then - LOSS=$(tail -$WORKERS $ZEEKLOG | awk -F, '{print $NF}' | sed 's/}//' | awk -v WORKERS=$WORKERS -F: '{LOSS += $2 / WORKERS} END { print LOSS}') - echo "zeekcaptureloss loss=$LOSS" + if [ -d "/host/nsm/zeek/spool/logger" ]; then + WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} + ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log + elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then + WORKERS=1 + ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log + else + echo 'Zeek capture_loss.log not found' >/dev/stderr + exit 2 fi - fi - echo "$CURRENTTS" > $LASTCAPTURELOSSLOG + + LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt + if [ -f "$ZEEKLOG" ]; then + CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g') + if [ -f "$LASTCAPTURELOSSLOG" ]; then + LASTTS=$(cat $LASTCAPTURELOSSLOG) + if [[ "$LASTTS" != "$CURRENTTS" ]]; then + LOSS=$(tail -$WORKERS $ZEEKLOG | awk -F, '{print $NF}' | sed 's/}//' | awk -v WORKERS=$WORKERS -F: '{LOSS += $2 / WORKERS} END { print LOSS}') + echo "zeekcaptureloss loss=$LOSS" + fi + fi + echo "$CURRENTTS" > $LASTCAPTURELOSSLOG +else + exit 0 fi diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index 0c1a714ba..559c6b15f 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -17,34 +17,31 @@ # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage -APP=zeekloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) -declare RESULT=($ZEEKLOG) -CURRENTDROP=${RESULT[3]} -# zeek likely not running if this is true -if [[ $CURRENTDROP == "rcvd:" ]]; then - CURRENTDROP=0 - PASTDROP=0 - DROPPED=0 +if [ ! $THEGREP ]; then + + ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) + declare RESULT=($ZEEKLOG) + CURRENTDROP=${RESULT[3]} + # zeek likely not running if this is true + if [[ $CURRENTDROP == "rcvd:" ]]; then + CURRENTDROP=0 + PASTDROP=0 + DROPPED=0 + else + PASTDROP=${RESULT[9]} + DROPPED=$((CURRENTDROP - PASTDROP)) + fi + if [[ "$DROPPED" -le 0 ]]; then + LOSS=0 + echo "zeekdrop drop=0" + else + CURRENTPACKETS=${RESULT[5]} + PASTPACKETS=${RESULT[11]} + TOTAL=$((CURRENTPACKETS - PASTPACKETS)) + LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) + echo "zeekdrop drop=$LOSS" else - PASTDROP=${RESULT[9]} - DROPPED=$((CURRENTDROP - PASTDROP)) -fi -if [[ "$DROPPED" -le 0 ]]; then - LOSS=0 - echo "zeekdrop drop=0" -else - CURRENTPACKETS=${RESULT[5]} - PASTPACKETS=${RESULT[11]} - TOTAL=$((CURRENTPACKETS - PASTPACKETS)) - LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) - echo "zeekdrop drop=$LOSS" + exit 0 fi \ No newline at end of file From f5b04117728780797bdd1d7c50c9375fd6f10f27 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:11:19 -0400 Subject: [PATCH 189/258] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 1226c42e4..50cfeff91 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -40,6 +40,7 @@ if [ ! $THEGREP ]; then fi echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + fi else exit 0 From 54322f5e9d608d0f8feade99bd8b23ee29322a50 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:17:02 -0400 Subject: [PATCH 190/258] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 50cfeff91..22189e43a 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -17,7 +17,7 @@ THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/beatseps.txt' EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" From 0c0edbaac8168c240937571459a8fda19b73bccb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:29:46 -0400 Subject: [PATCH 191/258] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 2 +- salt/telegraf/scripts/checkfiles.sh | 4 ++-- salt/telegraf/scripts/eps.sh | 4 ++-- salt/telegraf/scripts/helixeps.sh | 4 ++-- salt/telegraf/scripts/influxdbsize.sh | 4 ++-- salt/telegraf/scripts/oldpcap.sh | 4 ++-- salt/telegraf/scripts/raid.sh | 4 ++-- salt/telegraf/scripts/redis.sh | 5 ++--- salt/telegraf/scripts/sostatus.sh | 4 ++-- salt/telegraf/scripts/stenoloss.sh | 4 ++-- salt/telegraf/scripts/suriloss.sh | 4 ++-- salt/telegraf/scripts/zeekcaptureloss.sh | 4 ++-- salt/telegraf/scripts/zeekloss.sh | 4 ++-- 13 files changed, 25 insertions(+), 26 deletions(-) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 22189e43a..5e8256c22 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 12cf3ece6..1d7a44382 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 99f001552..25332e94a 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/eps.txt' EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index 7922a7ab6..d24f1d1e7 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/helixevents.txt' EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" diff --git a/salt/telegraf/scripts/influxdbsize.sh b/salt/telegraf/scripts/influxdbsize.sh index f2ed41f35..46e230a8a 100644 --- a/salt/telegraf/scripts/influxdbsize.sh +++ b/salt/telegraf/scripts/influxdbsize.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index d43f16d14..f23c0c83f 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then # Get the data OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index c0aabe75f..03e309c38 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then if [ -f "$RAIDLOG" ]; then echo "raid $RAIDSTATUS" diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index 613e6cdea..b448bba2d 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -15,10 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -THEGREP=$(ps -ef | grep $0 | grep -v grep) - -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh index 1baf38d2b..a7222b67d 100644 --- a/salt/telegraf/scripts/sostatus.sh +++ b/salt/telegraf/scripts/sostatus.sh @@ -14,9 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then SOSTATUSLOG=/var/log/sostatus/status.log SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index a5c974a73..028637e16 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then TSFILE=/var/log/telegraf/laststenodrop.log if [ -f "$TSFILE" ]; then diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 0ae867b29..1f43fbaf8 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -16,9 +16,9 @@ # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 995971b18..03c9188ea 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -18,9 +18,9 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index 559c6b15f..c9bc843cf 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -17,9 +17,9 @@ # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) declare RESULT=($ZEEKLOG) From 59247b4579db88a914a274060a0e259fb42df9f8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 13:45:01 -0400 Subject: [PATCH 192/258] Add exit check to new menu --- setup/so-whiptail | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 06a1afec1..35c9f53ab 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1086,6 +1086,10 @@ whiptail_net_method() { local option_count=$(( ${#options[@]} / 2 )) network_traffic=$(whiptail --title "Security Onion Setup" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + network_traffic=$(echo "${network_traffic^^}" | tr -d ' ' | tr '+' '_') } From 002fa990559f75f2ed6eb86036c2d3b0052daa24 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 13:47:50 -0400 Subject: [PATCH 193/258] Fix whiptail order --- setup/so-setup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 0667c99db..6353cb996 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -302,6 +302,10 @@ if ! [[ -f $install_opt_file ]]; then source "$net_init_file" fi + if [[ $is_minion ]]; then + collect_mngr_hostname + fi + if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then whiptail_management_interface_setup fi @@ -313,10 +317,6 @@ if ! [[ -f $install_opt_file ]]; then set_main_ip >> $setup_log 2>&1 compare_main_nic_ip - if [[ $is_minion ]]; then - collect_mngr_hostname - fi - reset_proxy if [[ -z $is_airgap ]]; then collect_net_method From 07e0ce563da0e154d7e439f3555d2f2a1e1fd800 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 13:50:30 -0400 Subject: [PATCH 194/258] Symmetrical spaces + remove useless logic --- setup/so-whiptail | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 35c9f53ab..b5cf52a68 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1040,14 +1040,11 @@ whiptail_net_method() { EOM local options=( - " Direct" "" + " Direct " "" ) local proxy_desc="proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment." if [[ $is_minion ]]; then - local mngr_article - if [[ $is_distmanager ]]; then mngr_article="this"; else mngr_article="the"; fi - read -r -d '' options_msg <<- EOM ${options_msg} @@ -1059,9 +1056,9 @@ whiptail_net_method() { EOM options+=( - " Direct + Manager" "" - " Proxy" "" - " Proxy + Manager" "" + " Direct + Manager " "" + " Proxy " "" + " Proxy + Manager " "" ) local height=25 else @@ -1071,7 +1068,7 @@ whiptail_net_method() { "Proxy" - ${proxy_desc} EOM options+=( - " Proxy" "" + " Proxy " "" ) local height=17 fi From ba9a45bd0f4e663f349ae2b857fd95ed42d4bfa8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 14:02:00 -0400 Subject: [PATCH 195/258] Split network init + ssh copy notices --- setup/so-setup | 18 ++++++------- setup/so-whiptail | 68 +++++++++++++++++++---------------------------- 2 files changed, 37 insertions(+), 49 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 6353cb996..f75e195b8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -203,7 +203,7 @@ if ! [[ -f $install_opt_file ]]; then if [[ $option == "CONFIGURENETWORK" ]]; then collect_hostname network_init_whiptail - whiptail_management_interface_setup + whiptail_network_init_notice network_init printf '%s\n' \ "MNIC=$MNIC" \ @@ -302,15 +302,8 @@ if ! [[ -f $install_opt_file ]]; then source "$net_init_file" fi - if [[ $is_minion ]]; then - collect_mngr_hostname - fi - - if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then - whiptail_management_interface_setup - fi - if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + whiptail_network_init_notice network_init fi @@ -323,14 +316,21 @@ if ! [[ -f $install_opt_file ]]; then [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi + if [[ $is_minion ]]; then + collect_mngr_hostname + fi + if [[ $is_minion ]]; then add_mngr_ip_to_hosts fi if [[ $is_minion ]]; then + whiptail_ssh_key_copy_notice copy_ssh_key >> $setup_log 2>&1 fi + + if [[ $is_minion ]] && ! (compare_versions); then info "Installer version mismatch, downloading correct version from manager" printf '%s\n' \ diff --git a/setup/so-whiptail b/setup/so-whiptail index b5cf52a68..392d46078 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -987,46 +987,6 @@ whiptail_management_nic() { } -whiptail_management_interface_setup() { - [ -n "$TESTING" ] && return - - local minion_msg - local msg - local line_count - - if [[ $is_minion ]]; then - line_count=11 - minion_msg="copy the ssh key for soremote to the manager. This will bring you to the command line temporarily to accept the manager's ECDSA certificate and enter the password for soremote" - else - line_count=9 - minion_msg="" - fi - - if [[ $is_iso ]]; then - if [[ $minion_msg != "" ]]; then - if [[ -f $net_init_file ]]; then - msg=$minion_msg - else - msg="initialize networking and $minion_msg" - fi - else - msg="initialize networking" - fi - else - msg=$minion_msg - fi - - read -r -d '' message <<- EOM - Setup will now $msg. - - Select OK to continue. - EOM - - whiptail --title "Security Onion Setup" --msgbox "$message" $line_count 75 - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - whiptail_net_method() { [ -n "$TESTING" ] && return @@ -1098,6 +1058,20 @@ whiptail_net_setup_complete() { exit 0 } +whiptail_network_init_notice() { + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + Setup will now initialize networking. + + Select OK to continue. + EOM + + whiptail --title "Security Onion Setup" --msgbox "$message" 9 75 + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_management_server() { [ -n "$TESTING" ] && return @@ -1760,6 +1734,20 @@ whiptail_so_allow() { whiptail_check_exitstatus $exitstatus } +whiptail_ssh_key_copy_notice() { + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + Setup will now copy the ssh key for soremote to the manager. This will bring you to the command line temporarily to accept the manager's ECDSA certificate and enter the password for soremote. + + Select OK to continue. + EOM + + whiptail --title "Security Onion Setup" --msgbox "$message" 11 75 + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whitpail_ssh_warning() { [ -n "$TESTING" ] && return From 7948906f51486a04d693e0f726f1cb348b7decba Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 14:04:01 -0400 Subject: [PATCH 196/258] Fix minion airgap logic --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index f75e195b8..4cb9c98a2 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -264,7 +264,7 @@ elif [ "$install_type" = 'ANALYST' ]; then fi # Check if this is an airgap install -if [[ $is_iso || $is_minion ]]; then +if [[ ( $is_manager || $is_import || $is_minion ) && $is_iso ]]; then whiptail_airgap if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true From d8457255cb357c6dac80eeaeb71c95cfa7df656f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 14:06:10 -0400 Subject: [PATCH 197/258] n -> z --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 7dd5511fb..73eef96c2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -483,7 +483,7 @@ collect_mtu() { collect_net_method() { whiptail_net_method - [[ -n $network_traffic ]] && collect_proxy + [[ -z $network_traffic ]] && collect_proxy if [[ "$network_traffic" == *"_MANAGER" ]]; then whiptail_manager_updates_warning From b37da027fd4c0d480110ec6e919bf5e34e94055c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 14:08:25 -0400 Subject: [PATCH 198/258] ECDSA to ED25519 --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 392d46078..ce6f0c112 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1738,7 +1738,7 @@ whiptail_ssh_key_copy_notice() { [ -n "$TESTING" ] && return read -r -d '' message <<- EOM - Setup will now copy the ssh key for soremote to the manager. This will bring you to the command line temporarily to accept the manager's ECDSA certificate and enter the password for soremote. + Setup will now copy the ssh key for soremote to the manager. This will bring you to the command line temporarily to accept the manager's ED25519 certificate and enter the password for soremote. Select OK to continue. EOM From d2067a42bdccc8b2f85bc2d55a45a516abe13f6f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 14:12:53 -0400 Subject: [PATCH 199/258] Don't skip new menu on airgap minions --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index ce6f0c112..a04636f6a 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -990,7 +990,7 @@ whiptail_management_nic() { whiptail_net_method() { [ -n "$TESTING" ] && return - [[ $is_airgap ]] && return + [[ $is_airgap && ! $is_minion ]] && return local pkg_mngr if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi From be6933e8fb80536be70bbf12697bbfeb958c6085 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 14:20:00 -0400 Subject: [PATCH 200/258] Change EPS for Telegraf --- salt/telegraf/scripts/checkfiles.sh | 2 +- salt/telegraf/scripts/eps.sh | 2 +- salt/telegraf/scripts/helixeps.sh | 1 + salt/telegraf/scripts/suriloss.sh | 1 + salt/telegraf/scripts/zeekcaptureloss.sh | 1 + salt/telegraf/scripts/zeekloss.sh | 1 + 6 files changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 1d7a44382..5c6ab56c1 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -19,7 +19,7 @@ THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then - FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) + FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l) echo "faffiles files=$FILES" else diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 25332e94a..b497c2519 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -39,7 +39,7 @@ if [ ! "$THEGREP" ]; then fi echo "consumptioneps eps=${EVENTS%%.*}" - + fi else exit 0 fi diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index d24f1d1e7..1411cc40b 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -38,6 +38,7 @@ if [ ! "$THEGREP" ]; then fi echo "helixeps eps=${EVENTS%%.*}" + fi else exit 0 fi \ No newline at end of file diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 1f43fbaf8..2d0a56106 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -42,6 +42,7 @@ if [ ! "$THEGREP" ]; then LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) echo "suridrop drop=$LOSS" fi + fi else echo "suridrop drop=0" fi \ No newline at end of file diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 03c9188ea..6cb2dd2e2 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -44,6 +44,7 @@ if [ ! "$THEGREP" ]; then fi fi echo "$CURRENTTS" > $LASTCAPTURELOSSLOG + fi else exit 0 fi diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index c9bc843cf..3dbd42833 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -42,6 +42,7 @@ if [ ! "$THEGREP" ]; then TOTAL=$((CURRENTPACKETS - PASTPACKETS)) LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) echo "zeekdrop drop=$LOSS" + fi else exit 0 fi \ No newline at end of file From b4499557115ccca8f80f41dc707e1493b83b0cb1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 19 Apr 2021 16:26:53 -0400 Subject: [PATCH 201/258] Proxy whiptail fixes * Don't try to set up proxy/manager proxy during network only flow * Fix logic to never show new menu on airgap, set MANAGERUPDATES to 1 on airgap minions --- setup/so-functions | 2 -- setup/so-setup | 5 +---- setup/so-whiptail | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 73eef96c2..198178d03 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -483,8 +483,6 @@ collect_mtu() { collect_net_method() { whiptail_net_method - [[ -z $network_traffic ]] && collect_proxy - if [[ "$network_traffic" == *"_MANAGER" ]]; then whiptail_manager_updates_warning MANAGERUPDATES=1 diff --git a/setup/so-setup b/setup/so-setup index 4cb9c98a2..aa6ce489e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -210,9 +210,6 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" > "$net_init_file" set_main_ip >> $setup_log 2>&1 compare_main_nic_ip - reset_proxy - collect_net_method - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 whiptail_net_setup_complete else true @@ -428,7 +425,7 @@ fi if [[ $is_airgap ]]; then PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-manual} - MANAGERUPDATES=${MANAGERUPDATES:-0} + [[ ! $is_minion ]] && MANAGERUPDATES=${MANAGERUPDATES:-0} || MANAGERUPDATES=${MANAGERUPDATES:-1} fi # Start user prompts diff --git a/setup/so-whiptail b/setup/so-whiptail index a04636f6a..03011e7c6 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -990,8 +990,6 @@ whiptail_management_nic() { whiptail_net_method() { [ -n "$TESTING" ] && return - [[ $is_airgap && ! $is_minion ]] && return - local pkg_mngr if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi From 5072c24134da383deda3c4dff8540fe71913f1f4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 20 Apr 2021 08:12:44 -0400 Subject: [PATCH 202/258] Adjust sostatus timers --- salt/common/init.sls | 2 +- salt/telegraf/etc/telegraf.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 37d6b6768..6ef841ea4 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -242,7 +242,7 @@ sostatus_log: '/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1': cron.present: - user: root - - minute: '*/5' + - minute: '*/1' - hour: '*' - daymonth: '*' - month: '*' diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index dac9bf60e..a7be4d8a2 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -681,7 +681,7 @@ ] data_format = "influx" timeout = "15s" - interval = "180s" + interval = "60s" # ## Commands array From bbf16d0f11b5da8b39ac1243dc68553ec3993245 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 11:34:17 -0400 Subject: [PATCH 203/258] Show airgap prompt within if statement + persist variable for node installs --- setup/so-setup | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index aa6ce489e..08d67d2da 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -260,19 +260,19 @@ elif [ "$install_type" = 'ANALYST' ]; then is_analyst=true fi -# Check if this is an airgap install -if [[ ( $is_manager || $is_import || $is_minion ) && $is_iso ]]; then - whiptail_airgap - if [[ "$INTERWEBS" == 'AIRGAP' ]]; then - is_airgap=true - fi -fi - if [[ $is_manager || $is_import ]]; then check_elastic_license fi if ! [[ -f $install_opt_file ]]; then + # Check if this is an airgap install + if [[ ( $is_manager || $is_import || $is_minion ) && $is_iso ]]; then + whiptail_airgap + if [[ "$INTERWEBS" == 'AIRGAP' ]]; then + is_airgap=true + fi + fi + if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone" elif [[ $is_fleet_standalone ]]; then @@ -326,8 +326,6 @@ if ! [[ -f $install_opt_file ]]; then copy_ssh_key >> $setup_log 2>&1 fi - - if [[ $is_minion ]] && ! (compare_versions); then info "Installer version mismatch, downloading correct version from manager" printf '%s\n' \ @@ -336,6 +334,7 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ "MSRVIP=$MSRVIP" \ + "is_airgap=$is_airgap" \ "NODE_DESCRIPTION=\"$NODE_DESCRIPTION\"" > "$install_opt_file" [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball From cd0a115ac71b9f7b9a2fd56221c11c30f12a2dd2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 12:55:00 -0400 Subject: [PATCH 204/258] Fix acng config and don't show changes when proxy string can exist in file --- salt/manager/files/acng/acng.conf | 4 ++-- salt/manager/init.sls | 2 ++ salt/repo/client/init.sls | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index df934643b..55a46e616 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -91,6 +91,6 @@ PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirr # MaxInresponsiveDlSize: 64000 # BadRedirDetectMime: text/html {% set proxy = salt['pillar.get']('manager:proxy') -%} -{{ if proxy }} +{% if proxy %} Proxy: {{ proxy }} -{{ endif }} +{% endif %} diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 1d21c95d3..91635eb59 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -61,6 +61,8 @@ acngcopyconf: file.managed: - name: /opt/so/conf/aptcacher-ng/etc/acng.conf - source: salt://manager/files/acng/acng.conf + - template: jinja + - show_changes: False # Install the apt-cacher-ng container so-aptcacherng: diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls index 5567caac2..cff28fb4a 100644 --- a/salt/repo/client/init.sls +++ b/salt/repo/client/init.sls @@ -63,6 +63,7 @@ yumconf: - source: salt://repo/client/files/centos/yum.conf.jinja - mode: 644 - template: jinja + - show_changes: False {% endif %} cleanyum: From 369c0b43f57766435c7184d18817978efeee27b3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 12:55:23 -0400 Subject: [PATCH 205/258] Further jinja fixes --- salt/manager/files/acng/acng.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index 55a46e616..993452b57 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -91,6 +91,6 @@ PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirr # MaxInresponsiveDlSize: 64000 # BadRedirDetectMime: text/html {% set proxy = salt['pillar.get']('manager:proxy') -%} -{% if proxy %} +{% if proxy -%} Proxy: {{ proxy }} -{% endif %} +{% endif -%} From 95bb757b033220ffcf3ebe662cb13675cdefdd9d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 13:12:55 -0400 Subject: [PATCH 206/258] Fix salt-master check --- setup/so-functions | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 198178d03..fcdb66c15 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -175,10 +175,8 @@ __check_so_status() { } __check_salt_master() { - local salt_master_status - salt_master_status=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" systemctl is-active --quiet salt-master) - [[ -z $salt_master_status ]] && salt_master_status=1 - return $salt_master_status + $sshcmd -i /root/.ssh/so.key soremote@"$MSRV" systemctl is-active --quiet salt-master + return $? } check_network_manager_conf() { From ca9ac46cd2957f6fe309cad27ab47bd27ce8a150 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 13:27:52 -0400 Subject: [PATCH 207/258] Add keypress instruction at end of summary --- setup/so-whiptail | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 03011e7c6..926487f5b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -616,6 +616,9 @@ whiptail_end_settings() { fi fi + __append_end_msg "" + __append_end_msg "Press TAB to select yes or no." + whiptail --title "The following options have been set, would you like to proceed?" --yesno "$end_msg" 24 75 --scrolltext local exitstatus=$? From 113e558a0538b62ae44fc1ef99f06b8707feef4f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 14:32:17 -0400 Subject: [PATCH 208/258] Set manager early for proxy config --- setup/so-setup | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 08d67d2da..e75bfd8c8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -307,20 +307,17 @@ if ! [[ -f $install_opt_file ]]; then set_main_ip >> $setup_log 2>&1 compare_main_nic_ip + if [[ $is_minion ]]; then + collect_mngr_hostname + add_mngr_ip_to_hosts + fi + reset_proxy if [[ -z $is_airgap ]]; then collect_net_method [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi - if [[ $is_minion ]]; then - collect_mngr_hostname - fi - - if [[ $is_minion ]]; then - add_mngr_ip_to_hosts - fi - if [[ $is_minion ]]; then whiptail_ssh_key_copy_notice copy_ssh_key >> $setup_log 2>&1 From 3d9042349561613ca8efdb46cc18ce08f5700f9d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 14:44:58 -0400 Subject: [PATCH 209/258] Fix summary message to preserve empty line --- setup/so-whiptail | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 926487f5b..09346828f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -616,10 +616,14 @@ whiptail_end_settings() { fi fi - __append_end_msg "" - __append_end_msg "Press TAB to select yes or no." + local msg + read -r -d '' msg <<-EOM + $end_msg - whiptail --title "The following options have been set, would you like to proceed?" --yesno "$end_msg" 24 75 --scrolltext + Press TAB to select yes or no. + EOM + + whiptail --title "The following options have been set, would you like to proceed?" --yesno "$msg" 24 75 --scrolltext local exitstatus=$? whiptail_check_exitstatus $exitstatus From 3f007f102686f7036a274bc8b1480a059cb55526 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 15:18:06 -0400 Subject: [PATCH 210/258] Disable fastestmirror during setup + soup --- salt/common/tools/sbin/so-common | 4 ++++ salt/common/tools/sbin/soup | 3 ++- setup/so-functions | 22 +++++++++++----------- setup/so-setup | 2 ++ 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 97e61e6e2..a425bf7bd 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -122,6 +122,10 @@ check_elastic_license() { fi } +disable_fastestmirror() { + sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf +} + elastic_license() { read -r -d '' message <<- EOM diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2a1ddab1c..f17c180b8 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -425,8 +425,9 @@ up_2.3.3X_to_2.3.50_repo() { if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import - if [ $is_airgap -eq 1 ]; then + echo "Disabling fastestmirror." + disable_fastestmirror echo "Deleting unneeded repo files." DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') diff --git a/setup/so-functions b/setup/so-functions index fcdb66c15..c6925b778 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2321,24 +2321,24 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos if [[ "$OS" == "centos" ]]; then - if [[ "$INTERWEBS" == "AIRGAP" ]]; then - echo "This is airgap I don't need to add this repo" + if [[ "$INTERWEBS" == "AIRGAP" ]]; then + echo "This is airgap I don't need to add this repo" else - mkdir -p /root/oldrepos - mv -v /etc/yum.repos.d/* /root/oldrepos/ + mkdir -p /root/oldrepos + mv -v /etc/yum.repos.d/* /root/oldrepos/ ls -la /etc/yum.repos.d/ - rm -rf /etc/yum.repos.d + rm -rf /etc/yum.repos.d yum clean all yum repolist all mkdir -p /etc/yum.repos.d - if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/repo/client/files/centos/securityonioncache.repo /etc/yum.repos.d/ - else - cp -f ../salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ - fi + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ../salt/repo/client/files/centos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ../salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ + fi fi else - echo "This is Ubuntu" + echo "This is Ubuntu" fi } diff --git a/setup/so-setup b/setup/so-setup index e75bfd8c8..d7dd3b660 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -634,6 +634,8 @@ echo "1" > /root/accept_changes set_progress_str 2 'Updating packages' # Import the gpg keys gpg_rpm_import >> $setup_log 2>&1 + info "Disabling fastestmirror" + [[ $OS == 'centos' ]] && disable_fastestmirror if [[ ! $is_airgap ]]; then securityonion_repo >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 From cc4c0923016c6b212172f04ec1063ee42f5eb6cb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 20 Apr 2021 15:44:35 -0400 Subject: [PATCH 211/258] Add Security Onion Repo --- salt/repo/client/files/centos/securityonion.repo | 9 ++++++++- salt/repo/client/files/centos/securityonioncache.repo | 11 +++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/salt/repo/client/files/centos/securityonion.repo b/salt/repo/client/files/centos/securityonion.repo index 53788f272..06eefdc88 100644 --- a/salt/repo/client/files/centos/securityonion.repo +++ b/salt/repo/client/files/centos/securityonion.repo @@ -68,4 +68,11 @@ gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ -protect=1 \ No newline at end of file +protect=1 + +[securityonion] +name=Security Onion Repo repo +baseurl=https://repo.securityonion.net/file/securityonion-repo/securityonion/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/securityonion.pub \ No newline at end of file diff --git a/salt/repo/client/files/centos/securityonioncache.repo b/salt/repo/client/files/centos/securityonioncache.repo index d683ab410..e4d47cb11 100644 --- a/salt/repo/client/files/centos/securityonioncache.repo +++ b/salt/repo/client/files/centos/securityonioncache.repo @@ -67,5 +67,12 @@ gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository -baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ -protect=1 \ No newline at end of file +baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh4_repo/ +protect=1 + +[securityonion] +name=Security Onion Repo +baseurl=https://repocache.securityonion.net/file/securityonion-repo/securityonion/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/securityonion.pub \ No newline at end of file From f14df24ddc93f94a345591a556e0725bd0ce088f Mon Sep 17 00:00:00 2001 From: bryant-treacle Date: Wed, 21 Apr 2021 11:49:29 -0400 Subject: [PATCH 212/258] Update threading.map.jinja --- salt/suricata/threading.map.jinja | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/salt/suricata/threading.map.jinja b/salt/suricata/threading.map.jinja index fb9e16d6b..16bffb165 100644 --- a/salt/suricata/threading.map.jinja +++ b/salt/suricata/threading.map.jinja @@ -1,4 +1,18 @@ -{% if salt['pillar.get']('sensor:suriprocs') %} +{% if salt['pillar.get']('sensor:suripins') %} + {% load_yaml as cpu_affinity%} +cpu-affinity: + - management-cpu-set: + cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ] # include only these cpus in affinity settings + - receive-cpu-set: + cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ] # include only these cpus in affinity settings + - worker-cpu-set: + cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ] + mode: "exclusive" + threads: {{ salt['pillar.get']('sensor:suripins')|length }} + prio: + default: "high" + {% endload %} +{% elif salt['pillar.get']('sensor:suriprocs') %} {% load_yaml as cpu_affinity%} cpu-affinity: - management-cpu-set: @@ -15,18 +29,4 @@ cpu-affinity: high: [ 3 ] default: "high" {% endload %} -{% elif salt['pillar.get']('sensor:suripins') %} - {% load_yaml as cpu_affinity%} -cpu-affinity: - - management-cpu-set: - cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ] # include only these cpus in affinity settings - - receive-cpu-set: - cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ] # include only these cpus in affinity settings - - worker-cpu-set: - cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ] - mode: "exclusive" - threads: {{ salt['pillar.get']('sensor:suripins')|length }} - prio: - default: "high" - {% endload %} -{% endif %} \ No newline at end of file +{% endif %} From 075ba0d83b4b156837a2f885bddaf1d3acafe7e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 20 Apr 2021 13:12:55 -0400 Subject: [PATCH 213/258] Fix salt-master check --- setup/so-functions | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a37867b5a..c1f61a768 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -175,10 +175,8 @@ __check_so_status() { } __check_salt_master() { - local salt_master_status - salt_master_status=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" systemctl is-active --quiet salt-master) - [[ -z $salt_master_status ]] && salt_master_status=1 - return $salt_master_status + $sshcmd -i /root/.ssh/so.key soremote@"$MSRV" systemctl is-active --quiet salt-master + return $? } check_network_manager_conf() { From 06ccad334b45f78eb9eb954fb06ac6e2aef311a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 14:43:15 -0400 Subject: [PATCH 214/258] Fix Security --- salt/repo/client/files/centos/securityonioncache.repo | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/repo/client/files/centos/securityonioncache.repo b/salt/repo/client/files/centos/securityonioncache.repo index e4d47cb11..f4ec6ef3f 100644 --- a/salt/repo/client/files/centos/securityonioncache.repo +++ b/salt/repo/client/files/centos/securityonioncache.repo @@ -67,12 +67,12 @@ gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository -baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh4_repo/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/wazuh4_repo/ protect=1 [securityonion] name=Security Onion Repo -baseurl=https://repocache.securityonion.net/file/securityonion-repo/securityonion/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/securityonion/ enabled=1 gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/securityonion.pub \ No newline at end of file +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/securityonion.pub From 7335611166e5ff480676b560c99bd2fb5fca68eb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 15:35:05 -0400 Subject: [PATCH 215/258] soup will now ask to update packages --- salt/common/tools/sbin/soup | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2a1ddab1c..ee1dd662d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -161,6 +161,34 @@ check_log_size_limit() { fi } +check_os_updates() { + # Check to see if there are OS updates + NEEDUPDATES="WWe have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." + if [[ $OS == 'ubuntu' ]]; then + OSUPDATES=$(apt list --upgradeable) + else + OSUPDATES=$(yum -q list updates) + fi + if [[ "$OSUPDATES" > 1 ]]; then + echo $NEEDUPDATES + echo "" + read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm + + if [[ "$confirm" == [cC] ]]; then + echo "Continuing without updating packages" + elif [[ "$confirm" == [uU] ]]; then + echo "Applying Grid Updates" + salt \* -b 5 state.apply patch.os + else + echo "Exiting soup" + exit 0 + fi + else + echo "Looks like you have an updated OS" + fi + +} + clean_dockers() { # Place Holder for cleaning up old docker images echo "Trying to clean up old dockers." @@ -632,7 +660,7 @@ else rm -rf $UPDATE_DIR clone_to_tmp fi - +check_os_updates echo "" echo "Verifying we have the latest soup script." verify_latest_update_script @@ -815,8 +843,6 @@ Please review the following for more information about the update process and re https://docs.securityonion.net/soup https://blog.securityonion.net -Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup. - Press Enter to continue or Ctrl-C to cancel. EOF From 13ad07cd8899dd853aa8ca315e52519aa8637857 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 15:41:58 -0400 Subject: [PATCH 216/258] soup will now ask to update packages --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ee1dd662d..6d282e78c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -163,7 +163,7 @@ check_log_size_limit() { check_os_updates() { # Check to see if there are OS updates - NEEDUPDATES="WWe have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." + NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." if [[ $OS == 'ubuntu' ]]; then OSUPDATES=$(apt list --upgradeable) else From 73001713e3b64e198a6faa395189f560f53cba43 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 15:51:28 -0400 Subject: [PATCH 217/258] soup will now ask to update packages --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6d282e78c..e41e80afe 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -165,9 +165,9 @@ check_os_updates() { # Check to see if there are OS updates NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." if [[ $OS == 'ubuntu' ]]; then - OSUPDATES=$(apt list --upgradeable) + OSUPDATES=$(apt list --upgradeable | wc -l) else - OSUPDATES=$(yum -q list updates) + OSUPDATES=$(yum -q list updates | wc -l) fi if [[ "$OSUPDATES" > 1 ]]; then echo $NEEDUPDATES From f3d663f0906f86e5b752cfbfcd2c21f924e4aaa6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 21 Apr 2021 15:59:37 -0400 Subject: [PATCH 218/258] Don't set yum/apt proxy if updating through manager --- setup/so-functions | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c6925b778..f8c78238d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2410,13 +2410,15 @@ set_proxy() { "}" > /root/.docker/config.json # Set proxy for package manager - if [ "$OS" = 'centos' ]; then - echo "proxy=$so_proxy" >> /etc/yum.conf - else - # Set it up so the updates roll through the manager - printf '%s\n'\ - "Acquire::http::Proxy \"$so_proxy\";"\ - "Acquire::https::Proxy \"$so_proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf + if [[ $MANAGERUPDATES != 1 ]]; then + if [ "$OS" = 'centos' ]; then + echo "proxy=$so_proxy" >> /etc/yum.conf + else + # Set it up so the updates roll through the manager + printf '%s\n'\ + "Acquire::http::Proxy \"$so_proxy\";"\ + "Acquire::https::Proxy \"$so_proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf + fi fi # Set global git proxy @@ -2684,10 +2686,8 @@ set_redirect() { set_updates() { if [ "$MANAGERUPDATES" = '1' ]; then if [ "$OS" = 'centos' ]; then - if [[ ! $is_airgap ]]; then - if ! grep -q "$MSRV" /etc/yum.conf; then - echo "proxy=http://$MSRV:3142" >> /etc/yum.conf - fi + if [[ ! $is_airgap ]] && ! ( grep -q "$MSRV" /etc/yum.conf); then + echo "proxy=http://$MSRV:3142" >> /etc/yum.conf fi else # Set it up so the updates roll through the manager From 5c4be5e1cd9699b92e3ecc36a4121c69ef14fdf0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 16:15:40 -0400 Subject: [PATCH 219/258] soup will now ask to update packages --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e41e80afe..51d9f9fe1 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -165,11 +165,11 @@ check_os_updates() { # Check to see if there are OS updates NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." if [[ $OS == 'ubuntu' ]]; then - OSUPDATES=$(apt list --upgradeable | wc -l) + OSUPDATES=$(apt list --upgradeable | grep -v "^Listing..." | grep -v "^docker-ce" | grep -v "^wazuh-" | grep -v "^salt-" | wc -l) else OSUPDATES=$(yum -q list updates | wc -l) fi - if [[ "$OSUPDATES" > 1 ]]; then + if [[ "$OSUPDATES" -gt 1 ]]; then echo $NEEDUPDATES echo "" read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm From c1ae7ff3b65376a27289134e1ba04e9b7d00f290 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 21 Apr 2021 16:18:20 -0400 Subject: [PATCH 220/258] Set proxy, replace when setting up yum for manager proxy --- setup/so-functions | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index f8c78238d..86ef60203 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2410,15 +2410,13 @@ set_proxy() { "}" > /root/.docker/config.json # Set proxy for package manager - if [[ $MANAGERUPDATES != 1 ]]; then - if [ "$OS" = 'centos' ]; then - echo "proxy=$so_proxy" >> /etc/yum.conf - else - # Set it up so the updates roll through the manager - printf '%s\n'\ - "Acquire::http::Proxy \"$so_proxy\";"\ - "Acquire::https::Proxy \"$so_proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf - fi + if [ "$OS" = 'centos' ]; then + echo "proxy=$so_proxy" >> /etc/yum.conf + else + # Set it up so the updates roll through the manager + printf '%s\n'\ + "Acquire::http::Proxy \"$so_proxy\";"\ + "Acquire::https::Proxy \"$so_proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf fi # Set global git proxy @@ -2687,7 +2685,11 @@ set_updates() { if [ "$MANAGERUPDATES" = '1' ]; then if [ "$OS" = 'centos' ]; then if [[ ! $is_airgap ]] && ! ( grep -q "$MSRV" /etc/yum.conf); then - echo "proxy=http://$MSRV:3142" >> /etc/yum.conf + if grep -q "proxy="; then + sed -i "s/proxy=.*/proxy=http:\/\/$MSRV:3142/" + else + echo "proxy=http://$MSRV:3142" >> /etc/yum.conf + fi fi else # Set it up so the updates roll through the manager From b5b0c262c11b40694b2d750af85bc305928c0411 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 16:25:41 -0400 Subject: [PATCH 221/258] soup will now ask to update packages --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 51d9f9fe1..a5416df43 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -169,7 +169,7 @@ check_os_updates() { else OSUPDATES=$(yum -q list updates | wc -l) fi - if [[ "$OSUPDATES" -gt 1 ]]; then + if [[ "$OSUPDATES" -gt 0 ]]; then echo $NEEDUPDATES echo "" read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm From 261e7f7fd978a1243b051d419b877d61ec22dcec Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 21 Apr 2021 16:29:24 -0400 Subject: [PATCH 222/258] sed and grep need input files --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 86ef60203..e1467191e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2685,8 +2685,8 @@ set_updates() { if [ "$MANAGERUPDATES" = '1' ]; then if [ "$OS" = 'centos' ]; then if [[ ! $is_airgap ]] && ! ( grep -q "$MSRV" /etc/yum.conf); then - if grep -q "proxy="; then - sed -i "s/proxy=.*/proxy=http:\/\/$MSRV:3142/" + if grep -q "proxy=" /etc/yum.conf; then + sed -i "s/proxy=.*/proxy=http:\/\/$MSRV:3142/" /etc/yum.conf else echo "proxy=http://$MSRV:3142" >> /etc/yum.conf fi From fa972ea110582f8da607c9cdd77abe83e14340ca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 16:37:13 -0400 Subject: [PATCH 223/258] soup will now ask to update packages --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index a5416df43..019420afe 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -178,7 +178,7 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - salt \* -b 5 state.apply patch.os + salt \* -b 5 state.apply patch.os queue=True else echo "Exiting soup" exit 0 From c337be8f4f9964f7cede4929437c8ef470a148f2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Apr 2021 17:27:52 -0400 Subject: [PATCH 224/258] send suricata compress to dev/null --- salt/suricata/cron/surilogcompress | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/cron/surilogcompress b/salt/suricata/cron/surilogcompress index 4fe43bbee..94434d545 100644 --- a/salt/suricata/cron/surilogcompress +++ b/salt/suricata/cron/surilogcompress @@ -1,6 +1,6 @@ #!/bin/bash # Gzip the eve logs -find /nsm/suricata/eve*.json -type f -printf '%T@\t%p\n' | sort -t $'\t' -g | head -n -1 | cut -d $'\t' -f 2 | xargs nice gzip +find /nsm/suricata/eve*.json -type f -printf '%T@\t%p\n' | sort -t $'\t' -g | head -n -1 | cut -d $'\t' -f 2 | xargs nice gzip >/dev/null 2>&1 # TODO Add stats log \ No newline at end of file From 071e5166b40c894a78c5063ba2a335dbc22168f5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 21 Apr 2021 17:57:02 -0400 Subject: [PATCH 225/258] Set package manager source in patch pillar for yum.conf --- salt/repo/client/files/centos/yum.conf.jinja | 2 +- setup/so-functions | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/repo/client/files/centos/yum.conf.jinja b/salt/repo/client/files/centos/yum.conf.jinja index 506036421..d8cb32de1 100644 --- a/salt/repo/client/files/centos/yum.conf.jinja +++ b/salt/repo/client/files/centos/yum.conf.jinja @@ -12,7 +12,7 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release clean_requirements_on_remove=1 -{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') -%} +{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', 'direct') == 'manager' -%} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 {% elif proxy -%} proxy={{ proxy }} diff --git a/setup/so-functions b/setup/so-functions index e1467191e..217f9f360 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1669,7 +1669,6 @@ manager_global() { " fleet_ip: 'N/A'"\ " sensoronikey: '$SENSORONIKEY'"\ " wazuh: $WAZUH"\ - " managerupdate: $MANAGERUPDATES"\ " imagerepo: '$IMAGEREPO'"\ " pipeline: 'redis'"\ "sensoroni:"\ @@ -1865,9 +1864,16 @@ patch_pillar() { local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls + if [[ $MANAGERUPDATES == 1 ]]; then + local source="manager" + else + local source="direct" + fi + printf '%s\n'\ "patch:"\ " os:"\ + " source: $source"\ " schedule_name: '$PATCHSCHEDULENAME'"\ " enabled: True"\ " splay: 300"\ From c297031f6b2d438a9a035dbbf07a3086a92f27cc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 21 Apr 2021 17:58:13 -0400 Subject: [PATCH 226/258] Surround scalar in single quotes --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 217f9f360..d9309f098 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1873,7 +1873,7 @@ patch_pillar() { printf '%s\n'\ "patch:"\ " os:"\ - " source: $source"\ + " source: '$source'"\ " schedule_name: '$PATCHSCHEDULENAME'"\ " enabled: True"\ " splay: 300"\ From fae72aa243689c9ce91c3b330e371a9ce690471f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Apr 2021 08:25:01 -0400 Subject: [PATCH 227/258] Roll back cluster changes --- salt/telegraf/etc/telegraf.conf | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index a7be4d8a2..af3474913 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -615,29 +615,18 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false -{% if TRUE_CLUSTER %} - {% if grains.role == 'so-manager' %} -[[inputs.elasticsearch]] - servers = ["https://{{ MANAGER }}:9200"] - insecure_skip_verify = true - local = false - cluster_health = true - cluster_stats = true - {% endif %} - -{% else %} # # Read stats from one or more Elasticsearch servers or clusters - {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} +{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] servers = ["https://{{ MANAGER }}:9200"] insecure_skip_verify = true - {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} +{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] insecure_skip_verify = true - {% endif %} {% endif %} + # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" From 1d8e06590225bfaceb097ecc96641e30f5aea2b4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 08:35:50 -0400 Subject: [PATCH 228/258] fix salt retries - https://github.com/Security-Onion-Solutions/securityonion/issues/3948 --- salt/ca/init.sls | 5 +- salt/ssl/init.sls | 120 +++++++++++++++++++++++++++------------------- 2 files changed, 75 insertions(+), 50 deletions(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 0d35c10c1..485b01d3e 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -43,8 +43,9 @@ pki_private_key: - require: - file: /etc/pki - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 x509_pem_entries: module.run: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 8d6c65bea..861d08fcd 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -68,8 +68,9 @@ removeesp12dir: - x509: /etc/pki/influxdb.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the talking to influxdb /etc/pki/influxdb.crt: @@ -86,8 +87,9 @@ removeesp12dir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/influxdb.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 influxkeyperms: file.managed: @@ -111,8 +113,9 @@ influxkeyperms: - x509: /etc/pki/redis.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/redis.crt: x509.certificate_managed: @@ -128,8 +131,9 @@ influxkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/redis.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 rediskeyperms: file.managed: @@ -153,8 +157,9 @@ rediskeyperms: - x509: /etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: @@ -175,8 +180,9 @@ rediskeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt" - onchanges: @@ -232,8 +238,9 @@ fbcrtlink: - x509: /etc/pki/registry.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the docker registry /etc/pki/registry.crt: @@ -250,8 +257,9 @@ fbcrtlink: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/registry.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 regkeyperms: file.managed: @@ -273,8 +281,9 @@ regkeyperms: - x509: /etc/pki/minio.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for minio /etc/pki/minio.crt: @@ -291,8 +300,9 @@ regkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 miniokeyperms: file.managed: @@ -315,8 +325,9 @@ miniokeyperms: - x509: /etc/pki/elasticsearch.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/elasticsearch.crt: x509.certificate_managed: @@ -332,8 +343,9 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: @@ -366,8 +378,9 @@ elasticp12perms: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -385,8 +398,9 @@ elasticp12perms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -409,8 +423,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -425,8 +440,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -456,8 +472,9 @@ fbcertdir: - x509: /opt/so/conf/filebeat/etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /opt/so/conf/filebeat/etc/pki/filebeat.crt: @@ -478,8 +495,9 @@ fbcertdir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /opt/so/conf/filebeat/etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Convert the key to pkcs#8 so logstash will work correctly. filebeatpkcs: @@ -520,8 +538,9 @@ chownfilebeatp8: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -539,8 +558,9 @@ chownfilebeatp8: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -563,8 +583,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -579,8 +600,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -606,8 +628,9 @@ fleetkeyperms: - x509: /etc/pki/elasticsearch.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/elasticsearch.crt: x509.certificate_managed: @@ -623,8 +646,9 @@ fleetkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: From 1519936e4427b5298e68dc9ed55fd2b8757056a5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 22 Apr 2021 08:37:49 -0400 Subject: [PATCH 229/258] Use correct pillar value in yum.conf template --- salt/repo/client/files/centos/yum.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/repo/client/files/centos/yum.conf.jinja b/salt/repo/client/files/centos/yum.conf.jinja index d8cb32de1..aed183ff0 100644 --- a/salt/repo/client/files/centos/yum.conf.jinja +++ b/salt/repo/client/files/centos/yum.conf.jinja @@ -12,7 +12,7 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release clean_requirements_on_remove=1 -{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', 'direct') == 'manager' -%} +{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('patch:os:source', 'direct') == 'manager' -%} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 {% elif proxy -%} proxy={{ proxy }} From 8f37b6b73be81139766ce0e57844ed23990d193a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Apr 2021 09:35:42 -0400 Subject: [PATCH 230/258] Make sure VIM is installed with correct settings --- salt/common/files/vimrc | 6 ++++++ salt/common/init.sls | 10 ++++++++++ 2 files changed, 16 insertions(+) create mode 100644 salt/common/files/vimrc diff --git a/salt/common/files/vimrc b/salt/common/files/vimrc new file mode 100644 index 000000000..4234cf772 --- /dev/null +++ b/salt/common/files/vimrc @@ -0,0 +1,6 @@ +" Activates filetype detection +filetype plugin indent on + +" Sets .sls files to use YAML syntax highlighting +autocmd BufNewFile,BufRead *.sls set syntax=yaml +set number \ No newline at end of file diff --git a/salt/common/init.sls b/salt/common/init.sls index 6ef841ea4..012fcdc9b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -69,6 +69,13 @@ salttmp: - group: 939 - makedirs: True +# VIM config +vimconfig: + file.managed: + - name: /root/.vimrc + - source: salt://common/files/vimrc + - replace: False + # Install common packages {% if grains['os'] != 'CentOS' %} commonpkgs: @@ -95,6 +102,8 @@ commonpkgs: - python3-mysqldb - python3-packaging - git + - vim + heldpackages: pkg.installed: - pkgs: @@ -133,6 +142,7 @@ commonpkgs: - lvm2 - openssl - git + - vim-enhanced heldpackages: pkg.installed: From 7176a4214bd1b4ee965393d3efecc14041a45111 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 22 Apr 2021 09:42:39 -0400 Subject: [PATCH 231/258] Add support for legacy grids --- salt/repo/client/files/centos/yum.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/repo/client/files/centos/yum.conf.jinja b/salt/repo/client/files/centos/yum.conf.jinja index aed183ff0..e89e365ab 100644 --- a/salt/repo/client/files/centos/yum.conf.jinja +++ b/salt/repo/client/files/centos/yum.conf.jinja @@ -12,7 +12,7 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release clean_requirements_on_remove=1 -{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('patch:os:source', 'direct') == 'manager' -%} +{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and ( salt['pillar.get']('global:managerupdate', '0') or salt['pillar.get']('patch:os:source', 'direct') == 'manager') ) -%} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 {% elif proxy -%} proxy={{ proxy }} From a41c40ccbbe7ae2ad1ebbf9d96485d3f6fafce85 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Apr 2021 10:53:59 -0400 Subject: [PATCH 232/258] Fix Zeek Setting for close-delete --- salt/common/tools/sbin/soup | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8c184e3d1..d6fb62822 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -475,6 +475,34 @@ up_2.3.3X_to_2.3.50_repo() { } up_2.3.3X_to_2.3.50() { + + cat < /tmp/supersed.txt +/so-zeek:/ { + p; + n; + /shards:/ { + p; + n; + /warm:/ { + p; + n; + /close:/ { + s/close: 365/close: 45/; + p; + n; + /delete:/ { + s/delete: 45/delete: 365/; + p; + d; + } + } + } + } +} +p; +EOF + sed -n -i -f /tmp/supersed.txt /opt/so/saltstack/local/global.sls + rm /tmp/supersed.txt INSTALLEDVERSION=2.3.50 } From 41a5818bb7ee5dbd58f0a672a5043fb5234dea1f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 22 Apr 2021 11:26:15 -0400 Subject: [PATCH 233/258] Remove extra paren --- salt/repo/client/files/centos/yum.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/repo/client/files/centos/yum.conf.jinja b/salt/repo/client/files/centos/yum.conf.jinja index e89e365ab..b3731d7d7 100644 --- a/salt/repo/client/files/centos/yum.conf.jinja +++ b/salt/repo/client/files/centos/yum.conf.jinja @@ -12,7 +12,7 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release clean_requirements_on_remove=1 -{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and ( salt['pillar.get']('global:managerupdate', '0') or salt['pillar.get']('patch:os:source', 'direct') == 'manager') ) -%} +{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and ( salt['pillar.get']('global:managerupdate', '0') or salt['pillar.get']('patch:os:source', 'direct') == 'manager' ) -%} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 {% elif proxy -%} proxy={{ proxy }} From 440c546bb4332a83c7a05dea39d4089df2c9ef80 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 11:41:14 -0400 Subject: [PATCH 234/258] remove docker-ce.repo --- salt/repo/client/map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/repo/client/map.jinja b/salt/repo/client/map.jinja index ccfa1eae2..dedb96783 100644 --- a/salt/repo/client/map.jinja +++ b/salt/repo/client/map.jinja @@ -10,6 +10,7 @@ 'CentOS-Sources.repo', 'CentOS-Vault.repo', 'CentOS-x86_64-kernel.repo', + 'docker-ce.repo', 'epel.repo', 'epel-testing.repo', 'saltstack.repo', From a49d6a8d5c6624e4373a3b63985ed83b452c80cd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 11:47:53 -0400 Subject: [PATCH 235/258] apply highstate to minions instead of just salt.minion for soup if salt needs upgraded --- salt/common/tools/sbin/soup | 2 +- salt/salt/map.jinja | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d6fb62822..94621d6f3 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -823,7 +823,7 @@ if [ "$UPGRADESALT" == "1" ]; then if [ $is_airgap -eq 0 ]; then salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' cmd.run "yum clean all" fi - salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.apply salt.minion queue=True + salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.highstate queue=True echo "" fi diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5d6d980be..1ff13f040 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -13,9 +13,9 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} - {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion' %} + {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %} From 781ac0293c8af58f0b2d03f274c063b50c6a8b74 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 13:22:08 -0400 Subject: [PATCH 236/258] fix SALTNOTHELD for salt.minion --- salt/salt/map.jinja | 4 ++-- salt/salt/minion.sls | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 1ff13f040..cb24cc85e 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -3,10 +3,10 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} - {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} + {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %} {% else %} {% set SPLITCHAR = '-' %} - {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} + {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 5145da34b..633e1496d 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -11,7 +11,7 @@ include: {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} -{% if SALTNOTHELD == 0 %} +{% if SALTNOTHELD | int == 0 %} unhold_salt_packages: module.run: - pkg.unhold: @@ -29,7 +29,7 @@ install_salt_minion: {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} -{% if SALTNOTHELD == 1 %} +{% if SALTNOTHELD | int == 1 %} hold_salt_packages: module.run: - pkg.hold: From 94352c212f0611a0e5b945a2f47289df0e6382e1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Apr 2021 13:26:41 -0400 Subject: [PATCH 237/258] Fix so-playbook-sync --- salt/common/tools/sbin/so-playbook-sync | 2 +- salt/common/tools/sbin/so-sensor-clean | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-playbook-sync b/salt/common/tools/sbin/so-playbook-sync index a76d398cb..333dff58c 100755 --- a/salt/common/tools/sbin/so-playbook-sync +++ b/salt/common/tools/sbin/so-playbook-sync @@ -19,6 +19,6 @@ # Check to see if we are already running IS_RUNNING=$(ps aux | pgrep -f "so-playbook-sync" | wc -l) -[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0 +[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0 docker exec so-soctopus python3 playbook_play-sync.py diff --git a/salt/common/tools/sbin/so-sensor-clean b/salt/common/tools/sbin/so-sensor-clean index e62c3c4da..c1dd7d37d 100755 --- a/salt/common/tools/sbin/so-sensor-clean +++ b/salt/common/tools/sbin/so-sensor-clean @@ -116,7 +116,7 @@ clean() { # Check to see if we are already running IS_RUNNING=$(ps aux | pgrep -f "so-sensor-clean" | wc -l) -[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0 +[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do From 016837df289ada9dcf69e4f1d1abe4c99899962d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Apr 2021 13:36:52 -0400 Subject: [PATCH 238/258] sync soversion --- salt/common/files/soversion | 2 ++ salt/common/init.sls | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 salt/common/files/soversion diff --git a/salt/common/files/soversion b/salt/common/files/soversion new file mode 100644 index 000000000..15cd06b7a --- /dev/null +++ b/salt/common/files/soversion @@ -0,0 +1,2 @@ +{%- set VERSION = salt['pillar.get']('global:soversion') -%} +{{ VERSION }} \ No newline at end of file diff --git a/salt/common/init.sls b/salt/common/init.sls index 012fcdc9b..d8d5f5a77 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -278,6 +278,14 @@ backupdir: - daymonth: '*' - month: '*' - dayweek: '*' +{% else %} +sostatus_log: + file.managed: + - name: /etc/soversion + - source: salt://common/files/soversion + - mode: 644 + - template: jinja + {% endif %} # Manager daemon.json From 93148e4adcb74124819a413595bdbf6c5e8d6943 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Apr 2021 13:39:33 -0400 Subject: [PATCH 239/258] sync soversion --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index d8d5f5a77..33a8b9984 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -279,7 +279,7 @@ backupdir: - month: '*' - dayweek: '*' {% else %} -sostatus_log: +soversionfile: file.managed: - name: /etc/soversion - source: salt://common/files/soversion From b7c6110e574c8d416be8ee335b8951ef5e36d94c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Apr 2021 13:41:58 -0400 Subject: [PATCH 240/258] sync soversion --- salt/common/files/soversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/files/soversion b/salt/common/files/soversion index 15cd06b7a..194749710 100644 --- a/salt/common/files/soversion +++ b/salt/common/files/soversion @@ -1,2 +1,2 @@ {%- set VERSION = salt['pillar.get']('global:soversion') -%} -{{ VERSION }} \ No newline at end of file +{{ VERSION }} From e7d3369cef009192876bea061b340ed3fff29a82 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Apr 2021 14:17:38 -0400 Subject: [PATCH 241/258] Update soup --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 94621d6f3..81bf0c8ff 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -501,7 +501,7 @@ up_2.3.3X_to_2.3.50() { } p; EOF - sed -n -i -f /tmp/supersed.txt /opt/so/saltstack/local/global.sls + sed -n -i -f /tmp/supersed.txt /opt/so/saltstack/local/pillar/global.sls rm /tmp/supersed.txt INSTALLEDVERSION=2.3.50 } From 36bc4f4aa8f543537b13b4553737052d3ba2ae39 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 15:21:36 -0400 Subject: [PATCH 242/258] remove by package name not wildcard --- salt/salt/map.jinja | 2 ++ salt/salt/minion.sls | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index cb24cc85e..7b32b9841 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -4,9 +4,11 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %} + {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} + {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 633e1496d..1a172d851 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,6 +2,7 @@ {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} {% from 'salt/map.jinja' import SALTNOTHELD %} +{% from 'salt/map.jinja' import SALTPACKAGES %} {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} {% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} @@ -15,7 +16,10 @@ include: unhold_salt_packages: module.run: - pkg.unhold: - - name: 'salt-*' + - pkgs: +{% for package in SALTPACKAGES %} + - {{ package }} +{% endfor %} {% endif %} install_salt_minion: @@ -33,7 +37,10 @@ install_salt_minion: hold_salt_packages: module.run: - pkg.hold: - - name: 'salt-*' + - pkgs: +{% for package in SALTPACKAGES %} + - {{ package }} +{% endfor %} {% endif %} set_log_levels: From 90683a7e04ae7b32bb598d483ae28410f92ea4cf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 15:22:55 -0400 Subject: [PATCH 243/258] fix UPDATE_DIR var --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index a425bf7bd..c099f33af 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -171,7 +171,7 @@ gpg_rpm_import() { if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC="../salt/repo/client/files/centos/keys" else - local RPMKEYSLOC="$UPDATEDIR/salt/repo/client/files/centos/keys" + local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') From 4f545eefc2813e565bd8004c4350ebb189b4666b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 15:27:57 -0400 Subject: [PATCH 244/258] update preflight --- setup/so-preflight | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-preflight b/setup/so-preflight index 1ef840284..e58feebd2 100644 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -46,8 +46,7 @@ check_new_repos() { if [[ $OS == 'centos' ]]; then local repo_arr=( "https://download.docker.com/linux/centos/docker-ce.repo" - "https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3003/SALTSTACK-GPG-KEY.pub" - "https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" + "https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub" "https://download.docker.com/linux/ubuntu/gpg" "https://packages.wazuh.com/key/GPG-KEY-WAZUH" "https://packages.wazuh.com/3.x/yum/" From 7c7624c87eca4fec205525bdab38f40c5a33244a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Apr 2021 16:32:58 -0400 Subject: [PATCH 245/258] let remote nodes upgrade on their own time --- salt/common/tools/sbin/soup | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 81bf0c8ff..34e970bea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -818,13 +818,12 @@ unmount_update thehive_maint if [ "$UPGRADESALT" == "1" ]; then - echo "" - echo "Upgrading Salt on the remaining Security Onion nodes from $INSTALLEDSALTVERSION to $NEWSALTVERSION." if [ $is_airgap -eq 0 ]; then - salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' cmd.run "yum clean all" + echo "" + echo "Cleaning repos on remote Security Onion nodes." + salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all" + echo "" fi - salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.highstate queue=True - echo "" fi check_sudoers From f4606828c7aec1f0fdfa9c94e50f7d3424c46da6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 23 Apr 2021 10:42:14 -0400 Subject: [PATCH 246/258] Update MOTD with training link and simply customization commands --- salt/soc/files/soc/motd.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index 295329f39..ab9d6b843 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -1,6 +1,6 @@ ## Getting Started -New to Security Onion 2? Check out the [Online Help](/docs/) and [Cheatsheet](/docs/cheatsheet.pdf) to learn how to best utilize Security Onion to hunt for evil! Find them in the upper-right menu. +New to Security Onion 2? Check out the [Online Help](/docs/) and [Cheatsheet](/docs/cheatsheet.pdf) to learn how to best utilize Security Onion to hunt for evil! Find them in the upper-right menu. Also, watch our free Security Onion 2 Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website. If you're ready to dive-in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. Or navigate to the [Hunt](/#/hunt) interface to hunt for evil that the alerts might have missed! @@ -10,16 +10,18 @@ The release notes have moved to the upper-right menu. Click on the [What's New]( ## Customize This Space -Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. Visit [mardownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. +Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. Visit [markdownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. To customize this content, login to the manager via SSH and execute the following command: ```bash -cp -f /opt/so/saltstack/default/salt/soc/files/soc/motd.md /opt/so/saltstack/local/salt/soc/files/soc/motd.md +sudo cp /opt/so/saltstack/default/salt/soc/files/soc/motd.md /opt/so/saltstack/local/salt/soc/files/soc/ ``` -Now, edit the new file as desired. Finally, run this command: +and edit the new file as desired. + +Finally, run this command: ```bash -salt-call state.apply soc queue=True +sudo so-soc-restart ``` From fff12b423a8ca97b24f888af3fe031ffe688f918 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 23 Apr 2021 11:56:27 -0400 Subject: [PATCH 247/258] remove eps graph from manager and update to consumptioneps for standalone and managersearch --- salt/grafana/dashboards/manager/manager.json | 133 ------------------ .../managersearch/managersearch.json | 2 +- .../dashboards/standalone/standalone.json | 2 +- 3 files changed, 2 insertions(+), 135 deletions(-) diff --git a/salt/grafana/dashboards/manager/manager.json b/salt/grafana/dashboards/manager/manager.json index 2ce913155..2af54c136 100644 --- a/salt/grafana/dashboards/manager/manager.json +++ b/salt/grafana/dashboards/manager/manager.json @@ -4322,139 +4322,6 @@ "align": false, "alignLevel": null } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 31 - }, - "hiddenSeries": false, - "id": 76, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "EPS", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "esteps", - "orderByTime": "ASC", - "policy": "default", - "queryType": "randomWalk", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "eps" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Estimated EPS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "EPS", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } } ], "refresh": false, diff --git a/salt/grafana/dashboards/managersearch/managersearch.json b/salt/grafana/dashboards/managersearch/managersearch.json index 15bf3cc73..b46cdcc76 100644 --- a/salt/grafana/dashboards/managersearch/managersearch.json +++ b/salt/grafana/dashboards/managersearch/managersearch.json @@ -5157,7 +5157,7 @@ "type": "fill" } ], - "measurement": "esteps", + "measurement": "consumptioneps", "orderByTime": "ASC", "policy": "default", "queryType": "randomWalk", diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 60a5c6c6c..701e6c9f2 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -5562,7 +5562,7 @@ "type": "fill" } ], - "measurement": "esteps", + "measurement": "consumptioneps", "orderByTime": "ASC", "policy": "default", "queryType": "randomWalk", From f2628f2e5b7ed1aa800526ac88c0ad24a26dbc2d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Apr 2021 12:09:41 -0400 Subject: [PATCH 248/258] Prime the CentOS Repos --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index d9309f098..ecf7a153d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2342,6 +2342,7 @@ securityonion_repo() { else cp -f ../salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ fi + yum repolist all fi else echo "This is Ubuntu" From 4d0b06dfc777f972b1a933bbdac5729dd072b340 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 25 Apr 2021 11:01:21 -0400 Subject: [PATCH 249/258] Fix updates for airgap --- salt/common/tools/sbin/soup | 6 ++---- salt/repo/client/init.sls | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 34e970bea..89cc38c83 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -716,6 +716,8 @@ echo "" echo "Updating dockers to $NEWVERSION." if [ $is_airgap -eq 0 ]; then airgap_update_dockers + update_centos_repo + yum clean all else update_registry update_docker_containers "soup" @@ -737,10 +739,6 @@ preupgrade_changes_2.3.50_repo if [ "$UPGRADESALT" == "1" ]; then echo "Upgrading Salt" # Update the repo files so it can actually upgrade - if [ $is_airgap -eq 0 ]; then - update_centos_repo - yum clean all - fi upgrade_salt fi diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls index cff28fb4a..f5d94e241 100644 --- a/salt/repo/client/init.sls +++ b/salt/repo/client/init.sls @@ -16,8 +16,9 @@ airgap_repo: pkgrepo.managed: - humanname: Airgap Repo - baseurl: https://{{ MANAGER }}/repo - - gpgcheck: 1 + - gpgcheck: 0 - sslverify: 0 + {% endif %} # from airgap and common From 6135d89721947278d188368008482b931c644f8a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 25 Apr 2021 12:19:34 -0400 Subject: [PATCH 250/258] Prompt airgap to update --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 89cc38c83..a5a0501bc 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -718,6 +718,7 @@ if [ $is_airgap -eq 0 ]; then airgap_update_dockers update_centos_repo yum clean all + check_os_updates else update_registry update_docker_containers "soup" From bbe2f81cb622b48ba3412b34a74936c8199a09a3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 26 Apr 2021 08:53:58 -0400 Subject: [PATCH 251/258] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index caae65cfd..353e51866 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Security Onion 2.3.40 +## Security Onion 2.3.50 -Security Onion 2.3.40 is here! +Security Onion 2.3.50 is here! ## Screenshots From 0499b141ede284858b1266aaee156f3d7de56735 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 26 Apr 2021 09:20:03 -0400 Subject: [PATCH 252/258] 2.3.50 sig files --- VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-2.3.50.iso.sig | Bin 0 -> 543 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.3.50.iso.sig diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 774116411..bd29a864e 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,16 +1,16 @@ -### 2.3.40 ISO image built on 2021/03/22 +### 2.3.50 ISO image built on 2021/04/25 ### Download and Verify -2.3.40 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso +2.3.50 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso -MD5: FB72C0675F262A714B287BB33CE82504 -SHA1: E8F5A9AA23990DF794611F9A178D88414F5DA81C -SHA256: DB125D6E770F75C3FD35ABE3F8A8B21454B7A7618C2B446D11B6AC8574601070 +MD5: 8B74AF6F29DB156E3D467B25E1D46449 +SHA1: 99A0A96C5F206471E4F1D26A8A2D577A8ECDAED5 +SHA256: CA0EE3793FC1356FB5C50D36107FA3BB39DE6C40EBE6C7C90075D5C189BB3083 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -24,22 +24,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.40.iso.sig securityonion-2.3.40.iso +gpg --verify securityonion-2.3.50.iso.sig securityonion-2.3.50.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 22 Mar 2021 09:35:50 AM EDT using RSA key ID FE507013 +gpg: Signature made Sun 25 Apr 2021 01:01:35 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.50.iso.sig b/sigs/securityonion-2.3.50.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..05d0d01628f4cc4ce32b0d49cb57937d6a64747f GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;7&pl<*Q2@re`V7LBIa1(075B(nuX|)-z2+BYNjp#JV zwG*X@!I*qu0;DHkt_(+G{1bx~=74ROQ|+^>iQ=eq30|oU$B|6zn7a;l6~_#8TWrFJ z#pv%-j>;urN7F1K>I?1*A3h$x+uX&u~; zR;Iz2peomHkFY@}XLP`^$fPQi7yfGmF#8Ut$kQ?DI!8UCUJSCv5%OeWD~!UF=RMOU zCyBZDsV1-9LZWO5u!M|T7|rLeiUNMIp?-aKzdvLfh5I_$j6nBM_jzszPS0A@mbQ-! z0&AK6zt_clEB5r5%p?#=7<^TH!vbk%Rmxak4^j?UR#<+G6rLq|_bMP$uupz){3HLK z1ziwXGXcyc$d*M;BgRN*+64@Kz6x;NpvxH^AeUiRFC2!>CjzZ363vg*q5pHIVn^h9 zq}nOvd;OV?mVma6M{!Eh{FV_g8jI}(?;WTg*3KlPQGS+4-IO#W;jD!mE2okD=jS|# zd@eLnP!+GyJteB893Q(0)}X}u;!nRY81Wpc!2CP2_ZD+%*zl-Ctm;*PE(C-QThdOj hcQvOU(|8|fj`^ZuqH_3q3de7e;9z+|B+g|eSJ)6+1Kj`s literal 0 HcmV?d00001 From df22269fc979ccdae7b484913ec36036463cf8c0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 26 Apr 2021 14:49:44 -0400 Subject: [PATCH 253/258] Repo Fix --- salt/common/tools/sbin/soup | 23 +++++++++++------------ salt/repo/client/init.sls | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index a5a0501bc..d6f0c4fa7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -453,19 +453,18 @@ up_2.3.3X_to_2.3.50_repo() { if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import + echo "Disabling fastestmirror." + disable_fastestmirror + echo "Deleting unneeded repo files." + DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') + + for DELREPO in "${DELREPOS[@]}"; do + if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then + echo "Deleting $DELREPO.repo" + rm -f "/etc/yum.repos.d/$DELREPO.repo" + fi + done if [ $is_airgap -eq 1 ]; then - echo "Disabling fastestmirror." - disable_fastestmirror - echo "Deleting unneeded repo files." - DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - - for DELREPO in "${DELREPOS[@]}"; do - if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then - echo "Deleting $DELREPO.repo" - rm -f "/etc/yum.repos.d/$DELREPO.repo" - fi - done - # Copy the new repo file if not airgap cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ yum clean all diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls index f5d94e241..105789eb9 100644 --- a/salt/repo/client/init.sls +++ b/salt/repo/client/init.sls @@ -16,7 +16,7 @@ airgap_repo: pkgrepo.managed: - humanname: Airgap Repo - baseurl: https://{{ MANAGER }}/repo - - gpgcheck: 0 + - gpgcheck: 1 - sslverify: 0 {% endif %} From 939414aef6e78af230de94346a2f8fa3bc6975cf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 26 Apr 2021 15:36:56 -0400 Subject: [PATCH 254/258] Prompt airgap to update --- salt/repo/client/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls index 105789eb9..f5d94e241 100644 --- a/salt/repo/client/init.sls +++ b/salt/repo/client/init.sls @@ -16,7 +16,7 @@ airgap_repo: pkgrepo.managed: - humanname: Airgap Repo - baseurl: https://{{ MANAGER }}/repo - - gpgcheck: 1 + - gpgcheck: 0 - sslverify: 0 {% endif %} From 167e656abb227a2bd1709ff9a5b519470b2a26c1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 26 Apr 2021 16:38:12 -0400 Subject: [PATCH 255/258] Repo Fix --- VERIFY_ISO.md | 10 +++++----- sigs/securityonion-2.3.50.iso.sig | Bin 543 -> 543 bytes 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index bd29a864e..faeb908a2 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,13 +1,13 @@ -### 2.3.50 ISO image built on 2021/04/25 +### 2.3.50 ISO image built on 2021/04/26 ### Download and Verify 2.3.50 ISO image: https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso -MD5: 8B74AF6F29DB156E3D467B25E1D46449 -SHA1: 99A0A96C5F206471E4F1D26A8A2D577A8ECDAED5 -SHA256: CA0EE3793FC1356FB5C50D36107FA3BB39DE6C40EBE6C7C90075D5C189BB3083 +MD5: 1FF774520D3B1323D83BBF90BD9EFACE +SHA1: 0F323335459A11850B68BB82E062F581225303EE +SHA256: 2AACD535E0EACE17E8DC7B560353D43111A287C59C23827612B720D742DFD994 Signature for ISO image: https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig @@ -39,7 +39,7 @@ gpg --verify securityonion-2.3.50.iso.sig securityonion-2.3.50.iso The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Sun 25 Apr 2021 01:01:35 PM EDT using RSA key ID FE507013 +gpg: Signature made Mon 26 Apr 2021 03:54:51 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.50.iso.sig b/sigs/securityonion-2.3.50.iso.sig index 05d0d01628f4cc4ce32b0d49cb57937d6a64747f..892dae61b593fea495d38eac1fec974f9c513e6c 100644 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;7)8jAo52@re`V7LBIa1&pv5C2S`i3JZF9-dZ`PcwCD z#HNUa|IL^Ly9ExkARnVhampob>u`G#z|ZsSEsm)Lb15v{6Ngzj@=KfzIf4{`LhmtU zBv_ZqjJN=1JiN7ZTlI&u-sxx4nwg?LsM@%0Ll|7HaAJ^443&ppy8Oh9K$J<$9INunYW0kos0a%}`>BJKAiBSU%N}_@3>l zx2xRsAs8HI8lZrXoP7~XSEUI#gHa7_cK;saqZ986KTjE}JQZ##^B%u4cPEjj6zJT| z_{YF^89(mgJ{ZG#|aY-jHNj&9cQTJbv7t*rIvP*DLRfTOM%{F8iw=Ngqi(wxh<_jvR9T1E~V z7|AjNJjz}(qD3%8qavZMC9yEH!!gZh;&Jt(t;n3LRUvxiU*n0%iaJ literal 543 zcmV+)0^t3L0vrSY0RjL91p;7&pl<*Q2@re`V7LBIa1(075B(nuX|)-z2+BYNjp#JV zwG*X@!I*qu0;DHkt_(+G{1bx~=74ROQ|+^>iQ=eq30|oU$B|6zn7a;l6~_#8TWrFJ z#pv%-j>;urN7F1K>I?1*A3h$x+uX&u~; zR;Iz2peomHkFY@}XLP`^$fPQi7yfGmF#8Ut$kQ?DI!8UCUJSCv5%OeWD~!UF=RMOU zCyBZDsV1-9LZWO5u!M|T7|rLeiUNMIp?-aKzdvLfh5I_$j6nBM_jzszPS0A@mbQ-! z0&AK6zt_clEB5r5%p?#=7<^TH!vbk%Rmxak4^j?UR#<+G6rLq|_bMP$uupz){3HLK z1ziwXGXcyc$d*M;BgRN*+64@Kz6x;NpvxH^AeUiRFC2!>CjzZ363vg*q5pHIVn^h9 zq}nOvd;OV?mVma6M{!Eh{FV_g8jI}(?;WTg*3KlPQGS+4-IO#W;jD!mE2okD=jS|# zd@eLnP!+GyJteB893Q(0)}X}u;!nRY81Wpc!2CP2_ZD+%*zl-Ctm;*PE(C-QThdOj hcQvOU(|8|fj`^ZuqH_3q3de7e;9z+|B+g|eSJ)6+1Kj`s From 458c3863777d9b33f17de454f5a71d4c61e6837e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 27 Apr 2021 13:37:37 -0400 Subject: [PATCH 256/258] Update import install --- salt/repo/client/files/centos/yum.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/repo/client/files/centos/yum.conf.jinja b/salt/repo/client/files/centos/yum.conf.jinja index b3731d7d7..8af48e99d 100644 --- a/salt/repo/client/files/centos/yum.conf.jinja +++ b/salt/repo/client/files/centos/yum.conf.jinja @@ -12,7 +12,7 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release clean_requirements_on_remove=1 -{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and ( salt['pillar.get']('global:managerupdate', '0') or salt['pillar.get']('patch:os:source', 'direct') == 'manager' ) -%} +{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import']) and ( salt['pillar.get']('global:managerupdate', '0') or salt['pillar.get']('patch:os:source', 'direct') == 'manager' ) -%} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 {% elif proxy -%} proxy={{ proxy }} From 03774e627046d0be031a3dafbab68862bee5448b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 27 Apr 2021 15:46:45 -0400 Subject: [PATCH 257/258] Repo Fix --- sigs/securityonion-2.3.50.iso.sig | Bin 543 -> 543 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/sigs/securityonion-2.3.50.iso.sig b/sigs/securityonion-2.3.50.iso.sig index 892dae61b593fea495d38eac1fec974f9c513e6c..d8405a0421e494f77a03bee88e5207e9f73c1e29 100644 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;7*RW$$#2@re`V7LBIa1$Um5CESxa#R+gq=PYL1RQjp zIUjoz5JIvAI&fTWU%o>h~+eB02ca8Fb5cDUR*+^E@ z`nMVQYSe6b-cVv-tuz_HMkKDGT76lmx(uDa5)|)sD?MLv{R* z1E#5Ie<@~K-Zay`EJ1)0fqS55MnW%AwIRWoN&_XyX$rPwU}^qQ1m*+fU}lY}1jLZC h24TfxOI&zaA^x(3X|swT8BU^s0u`G#z|ZsSEsm)Lb15v{6Ngzj@=KfzIf4{`LhmtU zBv_ZqjJN=1JiN7ZTlI&u-sxx4nwg?LsM@%0Ll|7HaAJ^443&ppy8Oh9K$J<$9INunYW0kos0a%}`>BJKAiBSU%N}_@3>l zx2xRsAs8HI8lZrXoP7~XSEUI#gHa7_cK;saqZ986KTjE}JQZ##^B%u4cPEjj6zJT| z_{YF^89(mgJ{ZG#|aY-jHNj&9cQTJbv7t*rIvP*DLRfTOM%{F8iw=Ngqi(wxh<_jvR9T1E~V z7|AjNJjz}(qD3%8qavZMC9yEH!!gZh;&Jt(t;n3LRUvxiU*n0%iaJ From b6792f73e0d645fd67fde0a105b194ab301a9c00 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 27 Apr 2021 15:51:30 -0400 Subject: [PATCH 258/258] Repo Fix --- VERIFY_ISO.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index faeb908a2..2b97521dd 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,13 +1,14 @@ -### 2.3.50 ISO image built on 2021/04/26 +### 2.3.50 ISO image built on 2021/04/27 + ### Download and Verify 2.3.50 ISO image: https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso -MD5: 1FF774520D3B1323D83BBF90BD9EFACE -SHA1: 0F323335459A11850B68BB82E062F581225303EE -SHA256: 2AACD535E0EACE17E8DC7B560353D43111A287C59C23827612B720D742DFD994 +MD5: C39CEA68B5A8AFC5CFFB2481797C0374 +SHA1: 00AD9F29ABE3AB495136989E62EBB8FA00DA82C6 +SHA256: D77AE370D7863837A989F6735413D1DD46B866D8D135A4C363B0633E3990387E Signature for ISO image: https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig @@ -39,7 +40,7 @@ gpg --verify securityonion-2.3.50.iso.sig securityonion-2.3.50.iso The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 26 Apr 2021 03:54:51 PM EDT using RSA key ID FE507013 +gpg: Signature made Tue 27 Apr 2021 02:17:25 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner.