From 50fa0dc81ae9e948d67ad09979fadd0b0372806b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Mar 2021 11:32:37 -0400 Subject: [PATCH 001/188] Allow user to enter a description during setup Resolves #2404 --- setup/so-setup | 10 ++++++++-- setup/so-whiptail | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..f20828b85 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -291,8 +291,13 @@ if ! [[ -f $install_opt_file ]]; then [[ -f $net_init_file ]] && whiptail_net_reinit && reinit_networking=true - if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then collect_hostname + fi + + whiptail_node_description + + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init_whiptail else source "$net_init_file" @@ -334,7 +339,8 @@ if ! [[ -f $install_opt_file ]]; then "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ - "MSRVIP=$MSRVIP" > "$install_opt_file" + "MSRVIP=$MSRVIP" \ + "NODE_DESCRIPTION=$NODE_DESCRIPTION" > "$install_opt_file" [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" diff --git a/setup/so-whiptail b/setup/so-whiptail index a0425b5af..bc002085c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1044,6 +1044,16 @@ whiptail_node_advanced() { } +whiptail_node_description() { + [ -n "$TESTING" ] && return + + NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup" \ + --inputbox "Enter a short description for the node or press ENTER to leave blank:" 10 75 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_node_es_heap() { [ -n "$TESTING" ] && return From 449e0d853ce36c2fb31fd3e35a8bc5cee0306f01 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Mar 2021 15:52:51 -0400 Subject: [PATCH 002/188] Initial support for ntp service via chronyd --- setup/so-functions | 57 ++++++++++++++++++++++++++++++++++++++++++++-- setup/so-setup | 5 ++++ setup/so-variables | 3 +++ setup/so-whiptail | 18 +++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 29a58e718..d5e8c0a6e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -486,6 +486,17 @@ collect_node_ls_pipeline_worker_count() { done } +collect_ntp_servers() { + if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' ]]; then + if whiptail_ntp_ask; then + [[ $is_airgap ]] && ntp_servers="" + whiptail_ntp_servers "$ntp_servers" + else + ntp_servers="" + fi + fi +} + collect_oinkcode() { whiptail_oinkcode @@ -702,6 +713,38 @@ configure_minion() { } >> "$setup_log" 2>&1 } +configure_ntp() { + local chrony_conf=/etc/chrony.conf + + # Install chrony if it isn't already installed + if command -v chronyc &> /dev/null; then + if [ "$OS" == centos ]; then + yum -y install chrony + else + retry 50 10 "apt-get -y install chrony" || exit 1 + fi + fi + + [[ -f $chrony_conf ]] && rm -f $chrony_conf + + # Build list of servers + for addr in "${ntp_servers[@]}"; do + echo "server $addr iburst" >> $chrony_conf + done + + printf '%s\n' \ + 'driftfile /var/lib/chrony/drift' \ + 'makestep 1.0 3' \ + 'rtcsync' \ + 'logdir /var/log/chrony' >> $chrony_conf + + systemctl enable chronyd + systemctl start chronyd + + # Sync time + chronyc -a makestep +} + checkin_at_boot() { local minion_config=/etc/salt/minion @@ -709,6 +752,12 @@ checkin_at_boot() { echo "startup_states: highstate" >> "$minion_config" } +check_ntp_configured() { + if systemctl is-active --quiet chronyd || systemctl is-active --quiet ntpd; then + ntp_configured=true + fi +} + check_requirements() { local standalone_or_dist=$1 local node_type=$2 # optional @@ -1564,12 +1613,16 @@ manager_global() { "global:"\ " soversion: '$SOVERSION'"\ " hnmanager: '$HNMANAGER'"\ - " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ - " managerip: '$MAINIP'" > "$global_pillar" + " managerip: '$MAINIP'" + " ntp_servers:" > "$global_pillar" + + for addr in "${ntp_servers[@]}"; do + echo " - '$addr'" >> "$global_pillar" + done if [[ $is_airgap ]]; then printf '%s\n'\ diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..2082653c5 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -534,6 +534,9 @@ if [[ $is_sensor && ! $is_eval ]]; then fi fi +check_ntp_configured +[[ -z $ntp_configured ]] || collect_ntp_servers + if [[ $is_node && ! $is_eval ]]; then whiptail_node_advanced if [ "$NODESETUP" == 'NODEADVANCED' ]; then @@ -581,6 +584,8 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' + [[ -z $ntp_configured ]] || [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 + reserve_ports set_path diff --git a/setup/so-variables b/setup/so-variables index a2fdf03c6..0a07fc79d 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -72,3 +72,6 @@ export install_opt_file net_init_file=/root/net_init export net_init_file + +ntp_servers="0.pool.ntp.org,1.pool.ntp.org" +export ntp_servers diff --git a/setup/so-whiptail b/setup/so-whiptail index a0425b5af..1ccdf6a90 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1105,6 +1105,24 @@ whiptail_node_ls_pipeline_worker() { } +whiptail_ntp_ask() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "Would you like to configure ntp servers?" 7 44 +} + +whiptail_ntp_servers() { + [ -n "$TESTING" ] && return + + ntp_string=$(whiptail --title "Security Onion Setup" \ + --inputbox "Input the NTP server(s) you would like to use, separated by commas:" 8 75 "$1" 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array +} + whiptail_oinkcode() { [ -n "$TESTING" ] && return From b3f558a1f8481a9144c2d96a50cb3ad8b0c147c9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:14:34 -0400 Subject: [PATCH 003/188] [fix] Also check if proxy is set before asking for ntp servers --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d5e8c0a6e..fd998da14 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -487,7 +487,7 @@ collect_node_ls_pipeline_worker_count() { } collect_ntp_servers() { - if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' ]]; then + if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then if whiptail_ntp_ask; then [[ $is_airgap ]] && ntp_servers="" whiptail_ntp_servers "$ntp_servers" From ace30c07ea5bb75489add30067bb802244764eca Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:22:09 -0400 Subject: [PATCH 004/188] [fix] Also sync time before updating system clock --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index fd998da14..a7a596abe 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -741,7 +741,8 @@ configure_ntp() { systemctl enable chronyd systemctl start chronyd - # Sync time + # Sync time & update the system time + chronyc -a 'burst 4/4' chronyc -a makestep } From 184c763b02d36e78024417cc31edfbe1b181d05f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:36:08 -0400 Subject: [PATCH 005/188] [fix] Export correct variable to check later in setup --- setup/so-functions | 2 +- setup/so-whiptail | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a7a596abe..a346128e1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -592,7 +592,7 @@ collect_proxy_details() { else so_proxy="$proxy_addr" fi - export proxy + export so_proxy fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index 1ccdf6a90..2743ab65b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1289,11 +1289,7 @@ whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - if [[ $arg != 'confirm' ]]; then - proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) - else - proxy_pass_confirm=$(whiptail --title "Security Onion Setup" --passwordbox "Please confirm the proxy password:" 8 60 3>&1 1>&2 2>&3) - fi + proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 0e9c81c145b1229bf82fc4976b55630b4a77e0aa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 09:44:44 -0400 Subject: [PATCH 006/188] Fix logic around ntp prompt --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 2082653c5..6ed3fa344 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -535,7 +535,7 @@ if [[ $is_sensor && ! $is_eval ]]; then fi check_ntp_configured -[[ -z $ntp_configured ]] || collect_ntp_servers +[[ -z $ntp_configured ]] && collect_ntp_servers if [[ $is_node && ! $is_eval ]]; then whiptail_node_advanced From 2d873b92fa3c19b43b5850220d65270536901a13 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 10:22:41 -0400 Subject: [PATCH 007/188] Fix ntp logic elsewhere --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 6ed3fa344..07eb49500 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -584,7 +584,7 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' - [[ -z $ntp_configured ]] || [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 + [[ -z $ntp_configured ]] && [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 reserve_ports From 9f0afd90f1852d28ad42d126f3cee187d3ef2115 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 11:27:37 -0400 Subject: [PATCH 008/188] [fix] Add missing backslash --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index a346128e1..dbc92aabc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1618,7 +1618,7 @@ manager_global() { " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ - " managerip: '$MAINIP'" + " managerip: '$MAINIP'"\ " ntp_servers:" > "$global_pillar" for addr in "${ntp_servers[@]}"; do From 3287a777a2203a775265c25ca85d9b34ab1acaf7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 11:41:12 -0400 Subject: [PATCH 009/188] [fix] Pre-fill hostname re-enter on default --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index dbc92aabc..ffaa079c1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -400,7 +400,7 @@ collect_hostname() { if [[ $HOSTNAME == 'securityonion' ]]; then # Will only check HOSTNAME=securityonion once if ! (whiptail_avoid_default_hostname); then - whiptail_set_hostname + whiptail_set_hostname "$HOSTNAME" fi fi From 23cd006724755dac1a8fb22e6dd0126d19d3a00c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Mar 2021 14:06:10 -0400 Subject: [PATCH 010/188] so-ssh-harden fixes * Change when script is run during setup * Add newlines to sshd config for legibility --- salt/common/tools/sbin/so-ssh-harden | 8 +++++++- setup/so-functions | 2 -- setup/so-setup | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 1fd7d58d9..5891e4309 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -38,8 +38,12 @@ print_msg() { fi } +# Add newline to ssh for legibility +echo "" >> /etc/ssh/sshd_config + if ! [[ $quiet ]]; then print_sshd_t "ciphers" "Before"; fi sshd -T | grep "^ciphers" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "ciphers" "After" echo "" @@ -51,6 +55,7 @@ fi if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "Before"; fi sshd -T | grep "^kexalgorithms" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "After" echo "" @@ -62,6 +67,7 @@ fi if ! [[ $quiet ]]; then print_sshd_t "macs" "Before"; fi sshd -T | grep "^macs" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "macs" "After" echo "" @@ -73,6 +79,7 @@ fi if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "Before"; fi sshd -T | grep "^hostkeyalgorithms" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g" >> /etc/ssh/sshd_config +echo "" >> /etc/ssh/sshd_config if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "After" echo "" @@ -90,4 +97,3 @@ fi {% if grains['os'] != 'CentOS' %} print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." {% endif %} - diff --git a/setup/so-functions b/setup/so-functions index 29a58e718..deeefb3ba 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1436,8 +1436,6 @@ install_cleanup() { info "Removing so-setup permission entry from sudoers file" sed -i '/so-setup/d' /etc/sudoers fi - - so-ssh-harden -q } import_registry_docker() { diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..ba76b303e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -888,6 +888,7 @@ set_redirect >> $setup_log 2>&1 set_progress_str 85 'Applying finishing touches' filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 + so-ssh-harden -q >> $setup_log 2>&1 if [[ $is_manager || $is_import ]]; then set_progress_str 87 'Adding user to SOC' From 5ade0b9f40c37dffe27d42c89a4afaddcac97f9a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 16:30:30 -0400 Subject: [PATCH 011/188] Implement customizable overview page --- salt/soc/files/soc/changes.json | 49 --------------------------------- salt/soc/files/soc/motd.md | 17 ++++++++++++ salt/soc/init.sls | 8 +++--- 3 files changed, 21 insertions(+), 53 deletions(-) delete mode 100644 salt/soc/files/soc/changes.json create mode 100644 salt/soc/files/soc/motd.md diff --git a/salt/soc/files/soc/changes.json b/salt/soc/files/soc/changes.json deleted file mode 100644 index dbc7b4061..000000000 --- a/salt/soc/files/soc/changes.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "title": "Security Onion 2.3.40 is here!", - "changes": [ - { "summary": "FEATURE: Add option for HTTP Method Specification/POST to Hunt/Alerts Actions #2904" }, - { "summary": "FEATURE: Add option to configure proxy for various tools used during setup + persist the proxy configuration #529" }, - { "summary": "FEATURE: Alerts/Hunt - Provide method for base64-encoding pivot value #1749" }, - { "summary": "FEATURE: Allow users to customize links in SOC #1248" }, - { "summary": "FEATURE: Display user who requested PCAP in SOC #2775" }, - { "summary": "FEATURE: Make SOC browser app connection timeouts adjustable #2408" }, - { "summary": "FEATURE: Move to FleetDM #3483" }, - { "summary": "FEATURE: Reduce field cache expiration from 1d to 5m, and expose value as a salt pillar #3537" }, - { "summary": "FEATURE: Refactor docker_clean salt state to use loop w/ inspection instead of hardcoded image list #3113" }, - { "summary": "FEATURE: Run so-ssh-harden during setup #1932" }, - { "summary": "FEATURE: SOC should only display links to tools that are enabled #1643" }, - { "summary": "FEATURE: Update Sigmac Osquery Field Mappings #3137" }, - { "summary": "FEATURE: User must accept the Elastic licence during setup #3233" }, - { "summary": "FEATURE: soup should output more guidance for distributed deployments at the end #3340" }, - { "summary": "FEATURE: soup should provide some initial information and then prompt the user to continue #3486" }, - { "summary": "FIX: Add cronjob for so-suricata-eve-clean script #3515" }, - { "summary": "FIX: Change Elasticsearch heap formula #1686" }, - { "summary": "FIX: Create a post install version loop in soup #3102" }, - { "summary": "FIX: Custom Kibana settings are not being applied properly on upgrades #3254" }, - { "summary": "FIX: Hunt query issues with quotes #3320" }, - { "summary": "FIX: IP Addresses don't work with .security #3327" }, - { "summary": "FIX: Improve DHCP leases query in Hunt #3395" }, - { "summary": "FIX: Improve Setup verbiage #3422" }, - { "summary": "FIX: Improve Suricata DHCP logging and parsing #3397" }, - { "summary": "FIX: Keep RELATED,ESTABLISHED rules at the top of iptables chains #3288" }, - { "summary": "FIX: Populate http.status_message field #3408" }, - { "summary": "FIX: Remove 'types removal' deprecation messages from elastic log. #3345" }, - { "summary": "FIX: Reword + fix formatting on ES data storage prompt #3205" }, - { "summary": "FIX: SMTP shoud read SNMP on Kibana SNMP view #3413" }, - { "summary": "FIX: Sensors can temporarily show offline while processing large PCAP jobs #3279" }, - { "summary": "FIX: Soup should log to the screen as well as to a file #3467" }, - { "summary": "FIX: Strelka port 57314 not immediately relinquished upon restart #3457" }, - { "summary": "FIX: Switch SOC to pull from fieldcaps API due to field caching changes in Kibana 7.11 #3502" }, - { "summary": "FIX: Syntax error in /etc/sysctl.d/99-reserved-ports.conf #3308" }, - { "summary": "FIX: Telegraf hardcoded to use https and is not aware of elasticsearch features #2061" }, - { "summary": "FIX: Zeek Index Close and Delete Count for curator #3274" }, - { "summary": "FIX: so-cortex-user-add and so-cortex-user-enable use wrong pillar value for api key #3388" }, - { "summary": "FIX: so-rule does not completely apply change #3289" }, - { "summary": "FIX: soup should recheck disk space after it tries to clean up. #3235" }, - { "summary": "UPGRADE: Elastic 7.11.2 #3389" }, - { "summary": "UPGRADE: Suricata 6.0.2 #3217" }, - { "summary": "UPGRADE: Zeek 4 #3216" }, - { "summary": "UPGRADE: Zeek container to use Python 3 #1113" }, - { "summary": "UPGRADE: docker-ce to latest #3493" } - ] -} \ No newline at end of file diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md new file mode 100644 index 000000000..54df73d1b --- /dev/null +++ b/salt/soc/files/soc/motd.md @@ -0,0 +1,17 @@ +## Getting Started + +New to Security Onion 2? Check out the [Online Help](/docs/) and [Cheatsheet](/docs/cheatsheet.pdf) to learn how to best utilize Security Onion to hunt for evil! Find them in the upper-right menu. + +If you're ready to dive-in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. Or navigate to the [Hunt](/#/hunt) interface to hunt for evil that the alerts might have missed! + +## What's New + +The release notes have moved to the upper-right menu. Click on the [What's New](/docs/release-notes.html) menu option to find all the latest fixes and features in this version of Security Onion! + +## Customize This Space + +Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. + +To customize this content, copy `/opt/so/saltstack/default/salt/soc/files/soc/motd.md` to `/opt/so/saltstack/local/salt/soc/files/soc/motd.md` and edit it as desired. + +Visit [mardownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. \ No newline at end of file diff --git a/salt/soc/init.sls b/salt/soc/init.sls index d31898e72..a2d3ecf89 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -35,10 +35,10 @@ socconfig: - mode: 600 - template: jinja -socchanges: +socmotd: file.managed: - - name: /opt/so/conf/soc/changes.json - - source: salt://soc/files/soc/changes.json + - name: /opt/so/conf/soc/motd.md + - source: salt://soc/files/soc/motd.md - user: 939 - group: 939 - mode: 600 @@ -61,7 +61,7 @@ so-soc: - binds: - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - - /opt/so/conf/soc/changes.json:/opt/sensoroni/html/changes.json:ro + - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} From 887920e7c53ae66524ab682c6d10e6ffb9284809 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 16:44:08 -0400 Subject: [PATCH 012/188] Implement customizable overview page --- salt/soc/files/soc/motd.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index 54df73d1b..43da6732a 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -10,8 +10,16 @@ The release notes have moved to the upper-right menu. Click on the [What's New]( ## Customize This Space -Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. +Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. Visit [mardownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. -To customize this content, copy `/opt/so/saltstack/default/salt/soc/files/soc/motd.md` to `/opt/so/saltstack/local/salt/soc/files/soc/motd.md` and edit it as desired. +To customize this content, login to the manager via SSH and execute the following command: -Visit [mardownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format. \ No newline at end of file +```bash +cp -f /opt/so/saltstack/default/salt/soc/files/soc/motd.md /opt/so/saltstack/local/salt/soc/files/soc/motd.md +``` + +Now, edit the new file as desired. Finally, run this command: + +```bash +salt-call state.apply soc queue=True +``` From 79ad87f83c4819353ba16bdf82f31ee472a83a2f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 21:16:17 -0400 Subject: [PATCH 013/188] Remove freqserver, minio, and domainstats from image list --- salt/common/tools/sbin/so-image-common | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 402ae97f3..be5a327f0 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -47,20 +47,17 @@ container_list() { TRUSTED_CONTAINERS=( "so-acng" "so-curator" - "so-domainstats" "so-elastalert" "so-elasticsearch" "so-filebeat" "so-fleet" "so-fleet-launcher" - "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" "so-kibana" "so-kratos" "so-logstash" - "so-minio" "so-mysql" "so-nginx" "so-pcaptools" From 982f2de33ce0e6025b49552345eb174f15652bc0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 24 Mar 2021 09:48:00 -0400 Subject: [PATCH 014/188] [fix] Refactor so-ssh-harden * Create a temp file to make changes, and only copy back over if any changes are made * Test changes as they're made, and exit if the test fails * Only add lines if they don't already exist in the config --- salt/common/tools/sbin/so-ssh-harden | 152 +++++++++++++++++---------- 1 file changed, 98 insertions(+), 54 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 5891e4309..0e0b28b39 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -6,14 +6,16 @@ if [[ $1 =~ ^(-q|--quiet) ]]; then quiet=true fi +sshd_config=/etc/ssh/sshd_config +temp_config=/tmp/sshd_config + before= after= reload_required=false -print_sshd_t() { +check_sshd_t() { local string=$1 local state=$2 - echo "${state}:" local grep_out grep_out=$(sshd -T | grep "^${string}") @@ -23,8 +25,17 @@ print_sshd_t() { else after=$grep_out fi +} - echo $grep_out +print_diff() { + local type=$1 + local diff + diff=$(diff -dqbB <(echo $before) <(echo $after)) + + if [[ -n $diff ]]; then + printf '%s\n' "$type" "$diff" + echo "" + fi } print_msg() { @@ -33,67 +44,100 @@ print_msg() { printf "%s\n" \ "----" \ "$msg" \ - "----" \ - "" + "----" fi } -# Add newline to ssh for legibility -echo "" >> /etc/ssh/sshd_config +add_if_missing() { + local string=$1 + if ! grep -q "$1" $temp_config; then + printf "%s\n\n" "$1" >> $temp_config + reload_required=true + fi +} -if ! [[ $quiet ]]; then print_sshd_t "ciphers" "Before"; fi -sshd -T | grep "^ciphers" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "ciphers" "After" - echo "" -fi +test_config() { + local msg + msg=$(sshd -t -f $temp_config) + local ret=$? -if [[ $before != $after ]]; then - reload_required=true -fi + if [[ -n $msg ]]; then + echo "Error found in temp sshd config:" + echo $msg + fi -if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "Before"; fi -sshd -T | grep "^kexalgorithms" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "kexalgorithms" "After" - echo "" -fi + return $ret +} -if [[ $before != $after ]]; then - reload_required=true -fi +main() { + if ! [[ $quiet ]]; then echo "Copying current config to $temp_config"; fi + cp $sshd_config $temp_config -if ! [[ $quiet ]]; then print_sshd_t "macs" "Before"; fi -sshd -T | grep "^macs" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "macs" "After" - echo "" -fi + # Add newline to ssh for legibility + echo "" >> $temp_config -if [[ $before != $after ]]; then - reload_required=true -fi + # Ciphers + check_sshd_t "ciphers" "Before" + local cipher_string + cipher_string=$(echo "$before" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g") -if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "Before"; fi -sshd -T | grep "^hostkeyalgorithms" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g" >> /etc/ssh/sshd_config -echo "" >> /etc/ssh/sshd_config -if ! [[ $quiet ]]; then - print_sshd_t "hostkeyalgorithms" "After" - echo "" -fi + check_sshd_t "ciphers" "After" -if [[ $before != $after ]]; then - reload_required=true -fi + if ! [[ $quiet ]]; then print_diff "ciphers"; fi -if [[ $reload_required == true ]]; then - print_msg "Reloading sshd to load config changes..." - systemctl reload sshd -fi + if [[ $before != $after ]]; then + add_if_missing "$cipher_string" && test_config || exit 1 + fi -{% if grains['os'] != 'CentOS' %} -print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." -{% endif %} + # KexAlgorithms + check_sshd_t "kexalgorithms" "Before" + + local kexalg_string + kexalg_string=$(echo "$before" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g") + + check_sshd_t "kexalgorithms" "After" + + if ! [[ $quiet ]]; then print_diff "kexalgorithms"; fi + + if [[ $before != $after ]]; then + add_if_missing "$kexalg_string" && test_config || exit 1 + fi + + # Macs + check_sshd_t "macs" "Before" + local macs_string + macs_string=$(echo "$before" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g") + + check_sshd_t "macs" "After" + + if ! [[ $quiet ]]; then print_diff "macs"; fi + + if [[ $before != $after ]]; then + add_if_missing "$mac_string" && test_config || exit 1 + fi + + # HostKeyAlgorithms + check_sshd_t "hostkeyalgorithms" "Before" + local hostkeyalg_string + hostkeyalg_string=$(echo "$before" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g") + + check_sshd_t "hostkeyalgorithms" "After" + + if ! [[ $quiet ]]; then print_diff "hostkeyalgorithms"; fi + + if [[ $before != $after ]]; then + add_if_missing "$hostkeyalg_string" && test_config || exit 1 + fi + + if [[ $reload_required == true ]]; then + mv -f $temp_config $sshd_config + if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes..."; fi + systemctl reload sshd + print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." + else + if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up."; fi + rm -f $temp_config + fi +} + +main From 08f46a779ac6b1ae2f5754b078a7238f4eb3b9aa Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 23 Mar 2021 21:16:17 -0400 Subject: [PATCH 015/188] Remove freqserver, minio, and domainstats from image list --- salt/common/tools/sbin/so-image-common | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 402ae97f3..be5a327f0 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -47,20 +47,17 @@ container_list() { TRUSTED_CONTAINERS=( "so-acng" "so-curator" - "so-domainstats" "so-elastalert" "so-elasticsearch" "so-filebeat" "so-fleet" "so-fleet-launcher" - "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" "so-kibana" "so-kratos" "so-logstash" - "so-minio" "so-mysql" "so-nginx" "so-pcaptools" From 7fc2467951c864627c0c3beff04b45259eb61f32 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 24 Mar 2021 15:00:02 -0400 Subject: [PATCH 016/188] Correct local online docs link to release notes --- salt/soc/files/soc/motd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index 43da6732a..295329f39 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -6,7 +6,7 @@ If you're ready to dive-in, take a look at the [Alerts](/#/alerts) interface to ## What's New -The release notes have moved to the upper-right menu. Click on the [What's New](/docs/release-notes.html) menu option to find all the latest fixes and features in this version of Security Onion! +The release notes have moved to the upper-right menu. Click on the [What's New](/docs/#document-release-notes) menu option to find all the latest fixes and features in this version of Security Onion! ## Customize This Space From af3951e1ad7dff3f28f03dbeb10466facf8d3cbb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Mar 2021 11:51:55 -0400 Subject: [PATCH 017/188] Attempt to use so repo for network install --- setup/so-functions | 49 ++++++++++++++++---------- setup/so-setup | 2 ++ setup/yum_repos/saltstack.repo | 6 ---- setup/yum_repos/securityonion.repo | 56 ++++++++++++++++++++++++++++++ setup/yum_repos/wazuh.repo | 7 ---- 5 files changed, 88 insertions(+), 32 deletions(-) delete mode 100644 setup/yum_repos/saltstack.repo create mode 100644 setup/yum_repos/securityonion.repo delete mode 100644 setup/yum_repos/wazuh.repo diff --git a/setup/so-functions b/setup/so-functions index 29a58e718..b6cf569fb 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1085,9 +1085,9 @@ docker_install() { if [ $OS = 'centos' ]; then { yum clean expire-cache; - if [[ ! $is_airgap ]]; then - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; - fi + #if [[ ! $is_airgap ]]; then + # yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; + #fi if [[ ! $is_iso ]]; then yum -y install docker-ce-20.10.5-3.el7 containerd.io-1.4.4-3.1.el7; fi @@ -1990,8 +1990,8 @@ saltify() { if [ $OS = 'centos' ]; then set_progress_str 5 'Installing Salt repo' { - sudo rpm --import https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/SALTSTACK-GPG-KEY.pub; - cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; + sudo rpm --import https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub; + #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; } >> "$setup_log" 2>&1 set_progress_str 6 'Installing various dependencies' if [[ ! $is_iso ]]; then @@ -2001,7 +2001,7 @@ saltify() { 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORT') reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then - logCmd "yum -y install epel-release" + #logCmd "yum -y install epel-release" logCmd "yum -y install sqlite argon2 curl mariadb-devel" fi # Download Ubuntu Keys in case manager updates = 1 @@ -2010,7 +2010,7 @@ saltify() { logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" logCmd "wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH" - logCmd "cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo" + #logCmd "cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo" fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then @@ -2019,29 +2019,29 @@ saltify() { systemctl enable salt-master >> "$setup_log" 2>&1 ;; *) - if [ "$MANAGERUPDATES" = '1' ]; then - { - if [[ ! $is_airgap ]]; then + #if [ "$MANAGERUPDATES" = '1' ]; then + # { + # if [[ ! $is_airgap ]]; then # Create the GPG Public Key for the Salt Repo - cp ./public_keys/salt.pem /etc/pki/rpm-gpg/saltstack-signing-key; + #cp ./public_keys/salt.pem /etc/pki/rpm-gpg/saltstack-signing-key; # Copy repo files over - cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; - else - info "This is airgap" - fi - } >> "$setup_log" 2>&1 - fi + #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; + # else + # info "This is airgap" + # fi + # } >> "$setup_log" 2>&1 + #fi ;; esac if [[ ! $is_airgap ]]; then - cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo >> "$setup_log" 2>&1 + #cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo >> "$setup_log" 2>&1 yum clean expire-cache >> "$setup_log" 2>&1 fi set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - yum -y install epel-release + #yum -y install epel-release yum -y install salt-minion-3002.5\ python3\ python36-docker\ @@ -2264,6 +2264,17 @@ secrets_pillar(){ fi } +securityonion_repo() { + # Remove all the current repos + if [ "$OS" = 'centos' ]; then + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + else + echo "This is Ubuntu" + fi +} + set_base_heapsizes() { es_heapsize ls_heapsize diff --git a/setup/so-setup b/setup/so-setup index 82e414ca4..a532158f4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -622,6 +622,8 @@ set_redirect >> $setup_log 2>&1 info "Creating airgap repo" create_repo >> $setup_log 2>&1 airgap_rules >> $setup_log 2>&1 + else + securityonion_repo >> $setup_log 2>&1 fi if [[ $is_minion ]]; then diff --git a/setup/yum_repos/saltstack.repo b/setup/yum_repos/saltstack.repo deleted file mode 100644 index 0430a62b8..000000000 --- a/setup/yum_repos/saltstack.repo +++ /dev/null @@ -1,6 +0,0 @@ -[saltstack] -name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/ -enabled=1 -gpgcheck=1 -gpgkey=https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/SALTSTACK-GPG-KEY.pub \ No newline at end of file diff --git a/setup/yum_repos/securityonion.repo b/setup/yum_repos/securityonion.repo new file mode 100644 index 000000000..20c907289 --- /dev/null +++ b/setup/yum_repos/securityonion.repo @@ -0,0 +1,56 @@ +[base] +name=CentOS-$releasever - Base +baseurl=https://repo.securityonion.net/file/securityonion-repo/base/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#released updates +[updates] +name=CentOS-$releasever - Updates +baseurl=https://repo.securityonion.net/file/securityonion-repo/updates/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that may be useful +[extras] +name=CentOS-$releasever - Extras +baseurl=https://repo.securityonion.net/file/securityonion-repo/extras/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that extend functionality of existing packages +[centosplus] +name=CentOS-$releasever - Plus +baseurl=https://repo.securityonion.net/file/securityonion-repo/centosplus/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +[epel] +name=Extra Packages for Enterprise Linux 7 - $basearch +baseurl=https://repo.securityonion.net/file/securityonion-repo/epel/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/RPM-GPG-KEY-EPEL-7 + +[docker-ce-stable] +name=Docker CE Stable - $basearch +baseurl=https://repo.securityonion.net/file/securityonion-repo/docker-ce-stable +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub + +[saltstack] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + +[wazuh_repo] +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 \ No newline at end of file diff --git a/setup/yum_repos/wazuh.repo b/setup/yum_repos/wazuh.repo deleted file mode 100644 index ae462c62f..000000000 --- a/setup/yum_repos/wazuh.repo +++ /dev/null @@ -1,7 +0,0 @@ -[wazuh_repo] -gpgcheck=1 -gpgkey=https://packages.wazuh.com/key/GPG-KEY-WAZUH -enabled=1 -name=Wazuh repository -baseurl=https://packages.wazuh.com/3.x/yum/ -protect=1 From 150e724a4a52f0e58a224f3d1f82311964118abb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 25 Mar 2021 13:37:54 -0400 Subject: [PATCH 018/188] Fix chrony install logic + add sleep for chrony to finish sync --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index ffaa079c1..c2ddb2125 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -717,7 +717,7 @@ configure_ntp() { local chrony_conf=/etc/chrony.conf # Install chrony if it isn't already installed - if command -v chronyc &> /dev/null; then + if ! command -v chronyc &> /dev/null; then if [ "$OS" == centos ]; then yum -y install chrony else @@ -743,6 +743,7 @@ configure_ntp() { # Sync time & update the system time chronyc -a 'burst 4/4' + sleep 20 # Wait for chrony to sync chronyc -a makestep } From eb674b3b938b3769ab5ae0886b80d7a6a462c29d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 25 Mar 2021 14:45:33 -0400 Subject: [PATCH 019/188] Validate list of ntp servers (ip4, hostname, or fqdn) --- salt/common/tools/sbin/so-common | 14 ++++++++++++++ setup/so-functions | 13 ++++++++++--- setup/so-setup | 2 +- setup/so-variables | 4 ++-- setup/so-whiptail | 2 -- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 676b908ce..340525272 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -419,6 +419,20 @@ valid_proxy() { [[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1 } +valid_ntp_list() { + local string=$1 + local ntp_arr + IFS="," read -r -a ntp_arr <<< "$string" + + for ntp in "${ntp_arr[@]}"; do + if ! valid_ip4 "$ntp" && ! valid_hostname "$ntp" && ! valid_fqdn "$ntp"; then + return 1 + fi + done + + return 0 +} + valid_string() { local str=$1 local min_length=${2:-1} diff --git a/setup/so-functions b/setup/so-functions index c2ddb2125..6dd10096b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -489,10 +489,17 @@ collect_node_ls_pipeline_worker_count() { collect_ntp_servers() { if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then if whiptail_ntp_ask; then - [[ $is_airgap ]] && ntp_servers="" - whiptail_ntp_servers "$ntp_servers" + [[ $is_airgap ]] && ntp_servers=() + whiptail_ntp_servers "$ntp_string" + + while ! valid_ntp_list "$ntp_string"; do + whiptail_invalid_input + whiptail_ntp_servers "$ntp_string" + done + + IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array else - ntp_servers="" + ntp_servers=() fi fi } diff --git a/setup/so-setup b/setup/so-setup index 07eb49500..d9e64105a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -584,7 +584,7 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' - [[ -z $ntp_configured ]] && [[ -n $ntp_servers ]] && configure_ntp >> $setup_log 2>&1 + [[ -z $ntp_configured ]] && [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 reserve_ports diff --git a/setup/so-variables b/setup/so-variables index 0a07fc79d..676cba4f0 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -73,5 +73,5 @@ export install_opt_file net_init_file=/root/net_init export net_init_file -ntp_servers="0.pool.ntp.org,1.pool.ntp.org" -export ntp_servers +ntp_string="0.pool.ntp.org,1.pool.ntp.org" +export ntp_string diff --git a/setup/so-whiptail b/setup/so-whiptail index 2743ab65b..00397a6fa 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1119,8 +1119,6 @@ whiptail_ntp_servers() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - - IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array } whiptail_oinkcode() { From 0195d366cc9b386f73e1aa47ce4c437fb354118b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 26 Mar 2021 14:44:27 -0400 Subject: [PATCH 020/188] Add custom banner to login page --- salt/nginx/etc/nginx.conf | 2 +- salt/soc/files/soc/banner.md | 0 salt/soc/init.sls | 10 ++++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 salt/soc/files/soc/banner.md diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 25e8bc11f..ea820442b 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -157,7 +157,7 @@ http { ssl_prefer_server_ciphers on; ssl_protocols TLSv1.2; - location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) { + location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*) { proxy_pass http://{{ manager_ip }}:9822; proxy_read_timeout 90; proxy_connect_timeout 90; diff --git a/salt/soc/files/soc/banner.md b/salt/soc/files/soc/banner.md new file mode 100644 index 000000000..e69de29bb diff --git a/salt/soc/init.sls b/salt/soc/init.sls index a2d3ecf89..18fda41da 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -44,6 +44,15 @@ socmotd: - mode: 600 - template: jinja +socbanner: + file.managed: + - name: /opt/so/conf/soc/banner.md + - source: salt://soc/files/soc/banner.md + - user: 939 + - group: 939 + - mode: 600 + - template: jinja + soccustom: file.managed: - name: /opt/so/conf/soc/custom.js @@ -62,6 +71,7 @@ so-soc: - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro + - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} From 8819cc1371ae9106e323f39c67cbe9b071d12d64 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:01:22 -0400 Subject: [PATCH 021/188] Fix acng to actually cache --- salt/airgap/init.sls | 2 +- salt/common/keys/GPG-KEY-WAZUH | 52 +++++++++++++++++++++++ salt/common/keys/RPM-GPG-KEY-EPEL-7 | 29 +++++++++++++ salt/common/keys/SALTSTACK-GPG-KEY.pub | 31 ++++++++++++++ salt/common/keys/docker.pub | 28 +++++++++++++ salt/common/keys/securityonion.pub | 52 +++++++++++++++++++++++ salt/common/tools/sbin/so-common | 17 ++++++++ salt/common/tools/sbin/soup | 1 + salt/manager/files/acng/acng.conf | 1 + setup/so-functions | 51 +++------------------- setup/so-setup | 4 +- setup/yum_repos/securityonioncache.repo | 56 +++++++++++++++++++++++++ 12 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 salt/common/keys/GPG-KEY-WAZUH create mode 100644 salt/common/keys/RPM-GPG-KEY-EPEL-7 create mode 100644 salt/common/keys/SALTSTACK-GPG-KEY.pub create mode 100644 salt/common/keys/docker.pub create mode 100644 salt/common/keys/securityonion.pub create mode 100644 setup/yum_repos/securityonioncache.repo diff --git a/salt/airgap/init.sls b/salt/airgap/init.sls index 818bb3a3b..4ff401099 100644 --- a/salt/airgap/init.sls +++ b/salt/airgap/init.sls @@ -11,7 +11,7 @@ airgap_repo: pkgrepo.managed: - humanname: Airgap Repo - baseurl: https://{{ MANAGER }}/repo - - gpgcheck: 0 + - gpgcheck: 1 - sslverify: 0 agbase: diff --git a/salt/common/keys/GPG-KEY-WAZUH b/salt/common/keys/GPG-KEY-WAZUH new file mode 100644 index 000000000..b424ccfae --- /dev/null +++ b/salt/common/keys/GPG-KEY-WAZUH @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFeeyYwBEACyf4VwV8c2++J5BmCl6ofLCtSIW3UoVrF4F+P19k/0ngnSfjWb +8pSWB11HjZ3Mr4YQeiD7yY06UZkrCXk+KXDlUjMK3VOY7oNPkqzNaP6+8bDwj4UA +hADMkaXBvWooGizhCoBtDb1bSbHKcAnQ3PTdiuaqF5bcyKk8hv939CHulL2xH+BP +mmTBi+PM83pwvR+VRTOT7QSzf29lW1jD79v4rtXHJs4KCz/amT/nUm/tBpv3q0sT +9M9rH7MTQPdqvzMl122JcZST75GzFJFl0XdSHd5PAh2mV8qYak5NYNnwA41UQVIa ++xqhSu44liSeZWUfRdhrQ/Nb01KV8lLAs11Sz787xkdF4ad25V/Rtg/s4UXt35K3 +klGOBwDnzPgHK/OK2PescI5Ve1z4x1C2bkGze+gk/3IcfGJwKZDfKzTtqkZ0MgpN +7RGghjkH4wpFmuswFFZRyV+s7jXYpxAesElDSmPJ0O07O4lQXQMROE+a2OCcm0eF +3+Cr6qxGtOp1oYMOVH0vOLYTpwOkAM12/qm7/fYuVPBQtVpTojjV5GDl2uGq7p0o +h9hyWnLeNRbAha0px6rXcF9wLwU5n7mH75mq5clps3sP1q1/VtP/Fr84Lm7OGke4 +9eD+tPNCdRx78RNWzhkdQxHk/b22LCn1v6p1Q0qBco9vw6eawEkz1qwAjQARAQAB +tDFXYXp1aC5jb20gKFdhenVoIFNpZ25pbmcgS2V5KSA8c3VwcG9ydEB3YXp1aC5j +b20+iQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheABQJZHNOBBQkU +SgzvAAoJEJaz7l8pERFF6xUP/3SbcmrI/u7a2EqZ0GxwQ/LRkPzWkJRnozCtNYHD +ZjiZgSB/+77hkPS0tsBK/GXFLKfJAuf13XFrCvEuI4Q/pLOCCKIGumKXItUIwJBD +HiEmVt/XxIijmlF7O1jcWqE/5CQXofjr03WMx+qzNabIwU/6dTKZN4FrR1jDk7yS +6FYBsbhVcSoqSpGYx7EcuK3c3sKKtnbacK2Sw3K9n8Wdj+EK83cbpMg8D/efVRqv +xypeCeojtY10y4bmugEwMYPgFkrSbicuiZc8NA8qhvFp6JFRq/uL0PGACyg05wB3 +S9U4wvSkmlo2/G74awna22UlaoYmSSz3UZdpWd2zBxflx17948QfTqyhO6bM8qLz +dSyR6/6olAcR1N+PBup8PoMdBte4ul/hJp8WIviW0AxJUTZSbVj5v/t43QAKEpCE +IMHvkK8PRHz/9kMd/2xN7LgMtihCrGZOnzErkjhlZvmiJ6kcJoD7ywzFnfJrntOU +DjNb3eqUFSEwmhD60Hd2OCkfmiV7NEE/YTd9B72NSwzj4Za/JUdlF64LMeIiHbYp +Lh7P+mR+lMJf/SWsQmlyuiQ2u8SY2aDFvzBS9WtpwiznuUdrbRN87+TYLSVqDifj +Ea3zOnzLaLYbOr6LHz1xbhAvInv7KLobgiw1E4WnBNWN8xVwVJLKNE7wV88k43XV +3L/RuQINBFeeyYwBEADD1Y3zW5OrnYZ6ghTd5PXDAMB8Z1ienmnb2IUzLM+i0yE2 +TpKSP/XYCTBhFa390rYgFO2lbLDVsiz7Txd94nHrdWXGEQfwrbxsvdlLLWk7iN8l +Fb4B60OfRi3yoR96a/kIPNa0x26+n79LtDuWZ/DTq5JSHztdd9F1sr3h8i5zYmtv +luj99ZorpwYejbBVUm0+gP0ioaXM37uO56UFVQk3po9GaS+GtLnlgoE5volgNYyO +rkeIua4uZVsifREkHCKoLJip6P7S3kTyfrpiSLhouEZ7kV1lbMbFgvHXyjm+/AIx +HIBy+H+e+HNt5gZzTKUJsuBjx44+4jYsOR67EjOdtPOpgiuJXhedzShEO6rbu/O4 +wM1rX45ZXDYa2FGblHCQ/VaS0ttFtztk91xwlWvjTR8vGvp5tIfCi+1GixPRQpbN +Y/oq8Kv4A7vB3JlJscJCljvRgaX0gTBzlaF6Gq0FdcWEl5F1zvsWCSc/Fv5WrUPY +5mG0m69YUTeVO6cZS1aiu9Qh3QAT/7NbUuGXIaAxKnu+kkjLSz+nTTlOyvbG7BVF +a6sDmv48Wqicebkc/rCtO4g8lO7KoA2xC/K/6PAxDrLkVyw8WPsAendmezNfHU+V +32pvWoQoQqu8ysoaEYc/j9fN4H3mEBCN3QUJYCugmHP0pu7VtpWwwMUqcGeUVwAR +AQABiQIlBBgBCAAPAhsMBQJZHNOaBQkUSg0HAAoJEJaz7l8pERFFhpkQAJ09mjjp +n9f18JGSMzP41fVucPuLBZ5XJL/hy2boII1FvgfmOETzNxLPblHdkJVjZS5iMrhL +EJ1jv+GQDtf68/0jO+HXuQIBmUJ53YwbuuQlLWH7CI2AxlSAKAn2kOApWMKsjnAv +JwS3eNGukOKWRfEKTqz2Vwi1H7M7ppypZ9keoyAoSIWb61gm7rXbfT+tVBetHfrU +EM5vz3AS3pJk6Yfqn10IZfiexXmsBD+SpJBNzMBsznCcWO2y4qZNLjFferBoizvV +34UnZyd1bkSN0T/MKp8sgJwqDJBS72tH6ZIM8NNoy29aPDkeaa8XlhkWiBdRizqL +BcxrV/1n3xdzfY9FX6s4KGudo+gYsVpY0mrpZU8jG8YUNLDXQTXnRo4CQOtRJJbA +RFDoZfsDqToZftuEhIsk+MaKlyXoA0eIYqGe6lXa/jEwvViqLYubCNLu0+kgNQ3v +hKF8Pf7eXFDAePw7guuvDvBOMQqBCaKCxsz1HoKRNYBEdUYrEQBJnX235Q4IsdI/ +GcQ/dvERJXaDCG8EPhnwc517EMUJDiJ1CxT4+VMHphmFbiVqmctz0upIj+D037Xk +CcgxNte6LZorGRZ/l1MYINliGJKtCCFK7XGVPKiJ8zyGSyPj1FfwtBy5hUX3aQtm +bvP0H2BRCKoelsbRENu58BkU6YhiUry7pVul +=SJij +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/RPM-GPG-KEY-EPEL-7 b/salt/common/keys/RPM-GPG-KEY-EPEL-7 new file mode 100644 index 000000000..f205ede46 --- /dev/null +++ b/salt/common/keys/RPM-GPG-KEY-EPEL-7 @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.11 (GNU/Linux) + +mQINBFKuaIQBEAC1UphXwMqCAarPUH/ZsOFslabeTVO2pDk5YnO96f+rgZB7xArB +OSeQk7B90iqSJ85/c72OAn4OXYvT63gfCeXpJs5M7emXkPsNQWWSju99lW+AqSNm +jYWhmRlLRGl0OO7gIwj776dIXvcMNFlzSPj00N2xAqjMbjlnV2n2abAE5gq6VpqP +vFXVyfrVa/ualogDVmf6h2t4Rdpifq8qTHsHFU3xpCz+T6/dGWKGQ42ZQfTaLnDM +jToAsmY0AyevkIbX6iZVtzGvanYpPcWW4X0RDPcpqfFNZk643xI4lsZ+Y2Er9Yu5 +S/8x0ly+tmmIokaE0wwbdUu740YTZjCesroYWiRg5zuQ2xfKxJoV5E+Eh+tYwGDJ +n6HfWhRgnudRRwvuJ45ztYVtKulKw8QQpd2STWrcQQDJaRWmnMooX/PATTjCBExB +9dkz38Druvk7IkHMtsIqlkAOQMdsX1d3Tov6BE2XDjIG0zFxLduJGbVwc/6rIc95 +T055j36Ez0HrjxdpTGOOHxRqMK5m9flFbaxxtDnS7w77WqzW7HjFrD0VeTx2vnjj +GqchHEQpfDpFOzb8LTFhgYidyRNUflQY35WLOzLNV+pV3eQ3Jg11UFwelSNLqfQf +uFRGc+zcwkNjHh5yPvm9odR1BIfqJ6sKGPGbtPNXo7ERMRypWyRz0zi0twARAQAB +tChGZWRvcmEgRVBFTCAoNykgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQI4BBMB +AgAiBQJSrmiEAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBqL66iNSxk +5cfGD/4spqpsTjtDM7qpytKLHKruZtvuWiqt5RfvT9ww9GUUFMZ4ZZGX4nUXg49q +ixDLayWR8ddG/s5kyOi3C0uX/6inzaYyRg+Bh70brqKUK14F1BrrPi29eaKfG+Gu +MFtXdBG2a7OtPmw3yuKmq9Epv6B0mP6E5KSdvSRSqJWtGcA6wRS/wDzXJENHp5re +9Ism3CYydpy0GLRA5wo4fPB5uLdUhLEUDvh2KK//fMjja3o0L+SNz8N0aDZyn5Ax +CU9RB3EHcTecFgoy5umRj99BZrebR1NO+4gBrivIfdvD4fJNfNBHXwhSH9ACGCNv +HnXVjHQF9iHWApKkRIeh8Fr2n5dtfJEF7SEX8GbX7FbsWo29kXMrVgNqHNyDnfAB +VoPubgQdtJZJkVZAkaHrMu8AytwT62Q4eNqmJI1aWbZQNI5jWYqc6RKuCK6/F99q +thFT9gJO17+yRuL6Uv2/vgzVR1RGdwVLKwlUjGPAjYflpCQwWMAASxiv9uPyYPHc +ErSrbRG0wjIfAR3vus1OSOx3xZHZpXFfmQTsDP7zVROLzV98R3JwFAxJ4/xqeON4 +vCPFU6OsT3lWQ8w7il5ohY95wmujfr6lk89kEzJdOTzcn7DBbUru33CQMGKZ3Evt +RjsC7FDbL017qxS+ZVA/HGkyfiu4cpgV8VUnbql5eAZ+1Ll6Dw== +=hdPa +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/SALTSTACK-GPG-KEY.pub b/salt/common/keys/SALTSTACK-GPG-KEY.pub new file mode 100644 index 000000000..14bd7d98c --- /dev/null +++ b/salt/common/keys/SALTSTACK-GPG-KEY.pub @@ -0,0 +1,31 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2 + +mQENBFOpvpgBCADkP656H41i8fpplEEB8IeLhugyC2rTEwwSclb8tQNYtUiGdna9 +m38kb0OS2DDrEdtdQb2hWCnswxaAkUunb2qq18vd3dBvlnI+C4/xu5ksZZkRj+fW +tArNR18V+2jkwcG26m8AxIrT+m4M6/bgnSfHTBtT5adNfVcTHqiT1JtCbQcXmwVw +WbqS6v/LhcsBE//SHne4uBCK/GHxZHhQ5jz5h+3vWeV4gvxS3Xu6v1IlIpLDwUts +kT1DumfynYnnZmWTGc6SYyIFXTPJLtnoWDb9OBdWgZxXfHEcBsKGha+bXO+m2tHA +gNneN9i5f8oNxo5njrL8jkCckOpNpng18BKXABEBAAG0MlNhbHRTdGFjayBQYWNr +YWdpbmcgVGVhbSA8cGFja2FnaW5nQHNhbHRzdGFjay5jb20+iQE4BBMBAgAiBQJT +qb6YAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAOCKFJ3le/vhkqB/0Q +WzELZf4d87WApzolLG+zpsJKtt/ueXL1W1KA7JILhXB1uyvVORt8uA9FjmE083o1 +yE66wCya7V8hjNn2lkLXboOUd1UTErlRg1GYbIt++VPscTxHxwpjDGxDB1/fiX2o +nK5SEpuj4IeIPJVE/uLNAwZyfX8DArLVJ5h8lknwiHlQLGlnOu9ulEAejwAKt9CU +4oYTszYM4xrbtjB/fR+mPnYh2fBoQO4d/NQiejIEyd9IEEMd/03AJQBuMux62tjA +/NwvQ9eqNgLw9NisFNHRWtP4jhAOsshv1WW+zPzu3ozoO+lLHixUIz7fqRk38q8Q +9oNR31KvrkSNrFbA3D89uQENBFOpvpgBCADJ79iH10AfAfpTBEQwa6vzUI3Eltqb +9aZ0xbZV8V/8pnuU7rqM7Z+nJgldibFk4gFG2bHCG1C5aEH/FmcOMvTKDhJSFQUx +uhgxttMArXm2c22OSy1hpsnVG68G32Nag/QFEJ++3hNnbyGZpHnPiYgej3FrerQJ +zv456wIsxRDMvJ1NZQB3twoCqwapC6FJE2hukSdWB5yCYpWlZJXBKzlYz/gwD/Fr +GL578WrLhKw3UvnJmlpqQaDKwmV2s7MsoZogC6wkHE92kGPG2GmoRD3ALjmCvN1E +PsIsQGnwpcXsRpYVCoW7e2nW4wUf7IkFZ94yOCmUq6WreWI4NggRcFC5ABEBAAGJ +AR8EGAECAAkFAlOpvpgCGwwACgkQDgihSd5Xv74/NggA08kEdBkiWWwJZUZEy7cK +WWcgjnRuOHd4rPeT+vQbOWGu6x4bxuVf9aTiYkf7ZjVF2lPn97EXOEGFWPZeZbH4 +vdRFH9jMtP+rrLt6+3c9j0M8SIJYwBL1+CNpEC/BuHj/Ra/cmnG5ZNhYebm76h5f +T9iPW9fFww36FzFka4VPlvA4oB7ebBtquFg3sdQNU/MmTVV4jPFWXxh4oRDDR+8N +1bcPnbB11b5ary99F/mqr7RgQ+YFF0uKRE3SKa7a+6cIuHEZ7Za+zhPaQlzAOZlx +fuBmScum8uQTrEF5+Um5zkwC7EXTdH1co/+/V/fpOtxIg4XO4kcugZefVm5ERfVS +MA== +=dtMN +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/docker.pub b/salt/common/keys/docker.pub new file mode 100644 index 000000000..1967cbf01 --- /dev/null +++ b/salt/common/keys/docker.pub @@ -0,0 +1,28 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFit5IEBEADDt86QpYKz5flnCsOyZ/fk3WwBKxfDjwHf/GIflo+4GWAXS7wJ +1PSzPsvSDATV10J44i5WQzh99q+lZvFCVRFiNhRmlmcXG+rk1QmDh3fsCCj9Q/yP +w8jn3Hx0zDtz8PIB/18ReftYJzUo34COLiHn8WiY20uGCF2pjdPgfxE+K454c4G7 +gKFqVUFYgPug2CS0quaBB5b0rpFUdzTeI5RCStd27nHCpuSDCvRYAfdv+4Y1yiVh +KKdoe3Smj+RnXeVMgDxtH9FJibZ3DK7WnMN2yeob6VqXox+FvKYJCCLkbQgQmE50 +uVK0uN71A1mQDcTRKQ2q3fFGlMTqJbbzr3LwnCBE6hV0a36t+DABtZTmz5O69xdJ +WGdBeePCnWVqtDb/BdEYz7hPKskcZBarygCCe2Xi7sZieoFZuq6ltPoCsdfEdfbO ++VBVKJnExqNZCcFUTEnbH4CldWROOzMS8BGUlkGpa59Sl1t0QcmWlw1EbkeMQNrN +spdR8lobcdNS9bpAJQqSHRZh3cAM9mA3Yq/bssUS/P2quRXLjJ9mIv3dky9C3udM ++q2unvnbNpPtIUly76FJ3s8g8sHeOnmYcKqNGqHq2Q3kMdA2eIbI0MqfOIo2+Xk0 +rNt3ctq3g+cQiorcN3rdHPsTRSAcp+NCz1QF9TwXYtH1XV24A6QMO0+CZwARAQAB +tCtEb2NrZXIgUmVsZWFzZSAoQ0UgcnBtKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 +BBMBCgAhBQJYrep4AhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEMUv62ti +Hp816C0P/iP+1uhSa6Qq3TIc5sIFE5JHxOO6y0R97cUdAmCbEqBiJHUPNQDQaaRG +VYBm0K013Q1gcJeUJvS32gthmIvhkstw7KTodwOM8Kl11CCqZ07NPFef1b2SaJ7l +TYpyUsT9+e343ph+O4C1oUQw6flaAJe+8ATCmI/4KxfhIjD2a/Q1voR5tUIxfexC +/LZTx05gyf2mAgEWlRm/cGTStNfqDN1uoKMlV+WFuB1j2oTUuO1/dr8mL+FgZAM3 +ntWFo9gQCllNV9ahYOON2gkoZoNuPUnHsf4Bj6BQJnIXbAhMk9H2sZzwUi9bgObZ +XO8+OrP4D4B9kCAKqqaQqA+O46LzO2vhN74lm/Fy6PumHuviqDBdN+HgtRPMUuao +xnuVJSvBu9sPdgT/pR1N9u/KnfAnnLtR6g+fx4mWz+ts/riB/KRHzXd+44jGKZra +IhTMfniguMJNsyEOO0AN8Tqcl0eRBxcOArcri7xu8HFvvl+e+ILymu4buusbYEVL +GBkYP5YMmScfKn+jnDVN4mWoN1Bq2yMhMGx6PA3hOvzPNsUoYy2BwDxNZyflzuAi +g59mgJm2NXtzNbSRJbMamKpQ69mzLWGdFNsRd4aH7PT7uPAURaf7B5BVp3UyjERW +5alSGnBqsZmvlRnVH5BDUhYsWZMPRQS9rRr4iGW0l+TH+O2VJ8aQ +=0Zqq +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/common/keys/securityonion.pub b/salt/common/keys/securityonion.pub new file mode 100644 index 000000000..c9148ff5c --- /dev/null +++ b/salt/common/keys/securityonion.pub @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF7rzwEBEADBg87uJhnC3Ls7s60hbHGaywGrPtbz2WuYA/ev3YS3X7WS75p8 +PGlzTWUCujx0pEHbK2vYfExl3zksZ8ZmLyZ9VB3oSLiWBzJgKAeB7YCFEo8te+eE +P2Z+8c+kX4eOV+2waxZyewA2TipSkhWgStSI4Ow8SyVUcUWA3hCw7mo2duNVi7KO +C3vvI3wzirH+8/XIGo+lWTg6yYlSxdf+0xWzYvV2QCMpwzJfARw6GGXtfCZw/zoO +o4+YPsiyztQdyI1y+g3Fbesl65E36DelbyP+lYd2VecX8ELEv0wlKCgHYlk6lc+n +qnOotVjWbsyXuFfo06PHUd6O9n3nmo0drC6kmXGw1e8hu0t8VcGfMTKS/hszwVUY +bHS6kbfsOoAb6LXPWKfqxk/BdreLXmcHHz88DimS3OS0JufkcmkjxEzSFRL0kb2h +QVb1SATrbx+v2RWQXvi9sLCjT2fdOiwi1Tgc84orc7A1C3Jwu353YaX9cV+n5uyG +OZ2AULZ5z2h13sVuiZAwfyyFs/O0CJ783hFA2TNPnyNGAgw/kaIo7nNRnggtndBo +oQzVS+BHiFx98IF4zDqmF2r2+jOCjxSrw8KnZBe4bgXFtl89DmjoejGvWDnu2MVM +pZDEs1DcOxHBQmTCWMIYLyNKG0xW6diyWBxEIaa7YgrP6kA+RaDfZ/xXPwARAQAB +tD9TZWN1cml0eSBPbmlvbiBTb2x1dGlvbnMsIExMQyA8aW5mb0BzZWN1cml0eW9u +aW9uc29sdXRpb25zLmNvbT6JAlQEEwEKAD4WIQTIBKk9Nr4Mcz6hlkR8EGC3/lBw +EwUCXuvPAQIbAwUJEswDAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRB8EGC3 +/lBwExB1D/42xIDGU2XFNFyTU+ZqzDA8qNC9hEKjLeizbeM8RIm3xO+3p7SdqbuJ +7pA8gk0RiHuILb+Ba1xiSh/w/W2bOxQhsXuWHih2z3W1tI+hu6RQhIm4e6CIHHf7 +Vzj4RSvHOVS0AzITUwkHjv0x0Z8zVBPJfEHKkK2x03BqP1o12rd7n2ZMrSfN6sED +fUwOJLDjthShtyLSPBVG8j7T5cfSCPSLhfVOKPQVcI1sSir7RLeyxt1v1kzjQdaA ++znxO8EgfZJN93wzfBrAGcVT8KmpmgwR6p46m20wJXyZC9DZxJ0o1y3toVWTC+kP +Qj1ROPivySVn10rBoOJk8HteyhW07gTcydq+noKHV7SqJ1899xRAYP7rDCfI9iMW +Nn22ZDLnAkIcbNR7JLJCHwsZH/Umo9KO/dIccIqVQel3UCCYZcWTZW0VkcjqVKRa +eK+JQGaJPrBAoxIG5/sMlbk2sINSubNWlcbH6kM0V8NVwdPiOO9xLmp2hI4ICxE3 +M+O2HCNX4QYzVizzTFxEvW3ieLa4nePQ8J6lvMI2oLkFP7xHoFluvZnuwfNvoEy0 +RnlHExN1UQTUvcbCxIbzjaJ4HJXilWHjgmGaVQO1S7AYskWnNWQ7uJvxnuZBNNwm +pIvwYEZp23fYaWl/xKqnmPMy2ADjROBKlCm7L+Ntq1r7ELGW5ZCTobkCDQRe688B +ARAA22GzdkSAo+mwJ2S1RbJ1G20tFnLsG/NC8iMN3lEh/PSmyPdB7mBtjZ+HPDzF +VSznXZdr3LItBBQOli2hVIj1lZBY7+s2ZufV3TFFwselUwT3b1g1KMkopD95Ckf8 +WhLbSz2yqgrvcEvbB0HFX/ZEsHGqIz2kLacixjwXXLWOMQ2LNbeW1f5zQkBnaNNQ +/4njzTj68OxnvfplNYNJqi2pZGb2UqarYX04FqKNuocN8E7AC9FQdBXylmVctw9T +pQVwfCI76bTe6vPWb+keb6UNN1jyXVnhIQ3Fv5sFBsmgXf/hO8tqCotrKjEiK2/i +RkvFeqsGMXreCgYg9zW4k+DcJtVa+Q8juGOjElrubY3Ua9mCusx3vY4QYSWxQ5Ih +k1lXiUcM5Rt38lfpKHRJ5Pd4Y5xlWSQfZ7nmzbf/GzJQz+rWrA0X6Oc6cDOPLNXK +w1dAygre4f2bsp5kHQt6NMefxeNTDmi+4R62K0tb40f5q0Vxz8qdyD48bBsbULNx +kb6mjOAD+FNkfNXcGeuTq9oRnjx8i93mhYsIP5LFNDXS/zSP1nv0ZUFeIlGQGjV9 +1wOvT454qkI9sKiVFtd4FrNKZJbKszxxDm+DPfB5j+hRC4oeEJ7w+sVyh3EawtfM +V7Mwj8i+7c3YUCravXBhSwG7SCTggFUgA8lMr8oWVgCATYsAEQEAAYkCPAQYAQoA +JhYhBMgEqT02vgxzPqGWRHwQYLf+UHATBQJe688BAhsMBQkSzAMAAAoJEHwQYLf+ +UHATTtwQAJiztPW68ykifpFdwYFp1VC7c+uGLhWBqjDY9NSUKNC9caR7bV0cnNu8 +07UG6j18gCB2GSkukXjOR/oTj6rNcW/WouPYfQOrw7+M2Ya8M8iq+E/HOXaXB3b4 +FeCcB0UuwfcHHd2KbXrRHA+9GNpmuOcfTCdsPpIr41Xg4QltATDEt/FrzuKspXg4 +vUKDXgfnbj7y0JcJM2FfcwWGlnAG5MMRyjJQAleGdiidX/9WxgJ4Mweq4qJM0jr3 +Qsrc9VuzxsLr85no3Hn5UYVgT7bBZ59HUbQoi775m78MxN3mWUSdcyLQKovI+YXr +tshTxWIf/2Ovdzt6Wq1WWXOGGuK1qgdPJTFWrlh3amFdb70zR1p6A/Lthd7Zty+n +QjRZRQo5jBSnYtjhMrZP6rxM3QqnQ0frEKK9HfDYONk1Bw18CUtdwFGb9OMregLR +IjvNLp9coSh5yYAepZyUGEPRET0GsmVw2trQF0uyMSkQfiq2zjPto6WWbsmrrbLr +cfZ/wnBw1FoNEd51U54euo9yvOgOVtJGvqLgHNwB8574FhQhoWAMhyizqdgeEt26 +m3FXecUNKL/AK71/l04vor+/WsXe8uhDg3O84qeYa9wgd8LZZVmGZJDosSwqYjtb +LdNNm+v60Zo6rFWSREegqi/nRTTDdxdW99ybjlh+mpbq3xavyFXF +=bhkm +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 676b908ce..326ad39da 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -162,6 +162,23 @@ get_random_value() { head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 } +gpg_rpm_import() { + if [ $OS = 'centos' ]; then + if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then + local RPMKEYSLOC=$temp_install_dir/salt/common/keys + else + local RPMKEYSLOC=$UPDATEDIR/salt/common/keys + fi + + RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') + + for RPMKEY in "${RPMKEYS[@]}"; do + rpm --import $RPMKEYSLOC/$RPMKEY + echo "Imported $RPMKEY" + done + fi +} + header() { printf '%s\n' "" "$banner" " $*" "$banner" } diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6ff298770..cb2d19aed 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -24,6 +24,7 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 SOUP_LOG=/root/soup.log +WHATWOULDYOUSAYYAHDOHERE=soup add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index a37d898af..1cc6bf6d9 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -20,6 +20,7 @@ Remap-npm: registry.npmjs.org Remap-node: nodejs.org Remap-apache: file:apache_mirrors ; file:backends_apache.us Remap-salt: repo.saltstack.com; https://repo.saltstack.com +Remap-securityonion: http://repocache.securityonion.net ; file:securityonion # Remap-secdeb: security.debian.org ReportPage: acng-report.html # SocketPath:/var/run/apt-cacher-ng/socket diff --git a/setup/so-functions b/setup/so-functions index b6cf569fb..d98a833da 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -159,11 +159,6 @@ check_network_manager_conf() { systemctl restart NetworkManager } >> "$setup_log" 2>&1 fi - - #if test -f "$nmconf"; then -# sed -i 's/managed=false/managed=true/g' "$nmconf" >> "$setup_log" 2>&1 -# systemctl restart NetworkManager >> "$setup_log" 2>&1 -# fi if [[ ! -d "$preupdir" ]]; then mkdir "$preupdir" >> "$setup_log" 2>&1 @@ -1054,40 +1049,11 @@ disable_ipv6() { } >> /etc/sysctl.conf } -#disable_misc_network_features() { -# filter_unused_nics -# if [ ${#filtered_nics[@]} -ne 0 ]; then -# for unused_nic in "${filtered_nics[@]}"; do -# if [ -n "$unused_nic" ]; then -# echo "Disabling unused NIC: $unused_nic" >> "$setup_log" 2>&1 -# -# # Disable DHCPv4/v6 and autoconnect -# nmcli con mod "$unused_nic" \ -# ipv4.method disabled \ -# ipv6.method ignore \ -# connection.autoconnect "no" >> "$setup_log" 2>&1 -# -# # Flush any existing IPs -# ip addr flush "$unused_nic" >> "$setup_log" 2>&1 -# fi -# done -# fi -# # Disable IPv6 -# { -# echo "net.ipv6.conf.all.disable_ipv6 = 1" -# echo "net.ipv6.conf.default.disable_ipv6 = 1" -# echo "net.ipv6.conf.lo.disable_ipv6 = 1" -# } >> /etc/sysctl.conf -#} - docker_install() { if [ $OS = 'centos' ]; then { yum clean expire-cache; - #if [[ ! $is_airgap ]]; then - # yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; - #fi if [[ ! $is_iso ]]; then yum -y install docker-ce-20.10.5-3.el7 containerd.io-1.4.4-3.1.el7; fi @@ -1988,11 +1954,6 @@ saltify() { # Install updates and Salt if [ $OS = 'centos' ]; then - set_progress_str 5 'Installing Salt repo' - { - sudo rpm --import https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub; - #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; - } >> "$setup_log" 2>&1 set_progress_str 6 'Installing various dependencies' if [[ ! $is_iso ]]; then logCmd "yum -y install wget nmap-ncat" @@ -2001,7 +1962,6 @@ saltify() { 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORT') reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then - #logCmd "yum -y install epel-release" logCmd "yum -y install sqlite argon2 curl mariadb-devel" fi # Download Ubuntu Keys in case manager updates = 1 @@ -2010,7 +1970,6 @@ saltify() { logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" logCmd "wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH" - #logCmd "cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo" fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then @@ -2035,13 +1994,11 @@ saltify() { ;; esac if [[ ! $is_airgap ]]; then - #cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo >> "$setup_log" 2>&1 yum clean expire-cache >> "$setup_log" 2>&1 fi set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - #yum -y install epel-release yum -y install salt-minion-3002.5\ python3\ python36-docker\ @@ -2266,10 +2223,14 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [ "$OS" = 'centos' ]; then + if [[ "$OS" == "centos" ]]; then mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ - cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ./yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + fi else echo "This is Ubuntu" fi diff --git a/setup/so-setup b/setup/so-setup index a532158f4..5989012ce 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -632,7 +632,9 @@ set_redirect >> $setup_log 2>&1 fi set_progress_str 2 'Updating packages' - update_packages >> $setup_log 2>&1 + if [[ ! $is_airgap ]] + update_packages >> $setup_log 2>&1 + fi if [[ $is_sensor || $is_helix || $is_import ]]; then set_progress_str 3 'Generating sensor pillar' diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo new file mode 100644 index 000000000..a38bac944 --- /dev/null +++ b/setup/yum_repos/securityonioncache.repo @@ -0,0 +1,56 @@ +[base] +name=CentOS-$releasever - Base +baseurl=https://repocache.securityonion.net/file/securityonion-repo/base/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#released updates +[updates] +name=CentOS-$releasever - Updates +baseurl=http://repocache.securityonion.net/file/securityonion-repo/updates/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that may be useful +[extras] +name=CentOS-$releasever - Extras +baseurl=http://repocache.securityonion.net/file/securityonion-repo/extras/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#additional packages that extend functionality of existing packages +[centosplus] +name=CentOS-$releasever - Plus +baseurl=http://repocache.securityonion.net/file/securityonion-repo/centosplus/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +[epel] +name=Extra Packages for Enterprise Linux 7 - $basearch +baseurl=http://repocache.securityonion.net/file/securityonion-repo/epel/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/RPM-GPG-KEY-EPEL-7 + +[docker-ce-stable] +name=Docker CE Stable - $basearch +baseurl=http://repocache.securityonion.net/file/securityonion-repo/docker-ce-stable +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub + +[saltstack] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +enabled=1 +gpgcheck=1 +gpgkey=https://repocache.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + +[wazuh_repo] +gpgcheck=1 +gpgkey=http://repocache.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 \ No newline at end of file From 26f8ae87c59c56dc621e9cfd5a8c763ca0851494 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:10:00 -0400 Subject: [PATCH 022/188] Fix acng to actually cache --- setup/so-setup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 5989012ce..edbb5b408 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -47,6 +47,7 @@ source ./so-variables # Parse command line arguments setup_type=$1 automation=$2 +WHATWOULDYOUSAYYAHDOHERE=setup while [[ $# -gt 0 ]]; do arg="$1" @@ -622,8 +623,6 @@ set_redirect >> $setup_log 2>&1 info "Creating airgap repo" create_repo >> $setup_log 2>&1 airgap_rules >> $setup_log 2>&1 - else - securityonion_repo >> $setup_log 2>&1 fi if [[ $is_minion ]]; then @@ -632,7 +631,11 @@ set_redirect >> $setup_log 2>&1 fi set_progress_str 2 'Updating packages' + # Import the gpg keys + gpg_rpm_import if [[ ! $is_airgap ]] + securityonion_repo >> $setup_log 2>&1 + gpg_rpm_import >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 fi From 955d41abde6522c5834e2c93b188a39921ff8690 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:18:49 -0400 Subject: [PATCH 023/188] Fix acng to actually cache --- salt/common/tools/sbin/so-common | 2 +- setup/so-setup | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 326ad39da..985042876 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -163,7 +163,7 @@ get_random_value() { } gpg_rpm_import() { - if [ $OS = 'centos' ]; then + if [[ "$OS" = "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC=$temp_install_dir/salt/common/keys else diff --git a/setup/so-setup b/setup/so-setup index edbb5b408..7d42f94bc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -635,7 +635,6 @@ set_redirect >> $setup_log 2>&1 gpg_rpm_import if [[ ! $is_airgap ]] securityonion_repo >> $setup_log 2>&1 - gpg_rpm_import >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 fi From bab062e52b7bd8cef5cae153edf0fb22c6063648 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Mar 2021 16:21:03 -0400 Subject: [PATCH 024/188] Fix acng to actually cache --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 985042876..53901a73a 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -163,7 +163,7 @@ get_random_value() { } gpg_rpm_import() { - if [[ "$OS" = "centos" ]]; then + if [[ "$OS" == "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC=$temp_install_dir/salt/common/keys else From 362bf555260b4141bb6dd796da6fdf10d12df582 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 28 Mar 2021 22:01:58 -0400 Subject: [PATCH 025/188] fixpath for GPG keys --- setup/so-functions | 1 + setup/so-setup | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d98a833da..99819e9a9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2226,6 +2226,7 @@ securityonion_repo() { if [[ "$OS" == "centos" ]]; then mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ./yum_repos/securityonioncache.repo /etc/yum.repos.d/ else diff --git a/setup/so-setup b/setup/so-setup index 7d42f94bc..9beb11cec 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -632,8 +632,8 @@ set_redirect >> $setup_log 2>&1 set_progress_str 2 'Updating packages' # Import the gpg keys - gpg_rpm_import - if [[ ! $is_airgap ]] + gpg_rpm_import >> $setup_log 2>&1 + if [[ ! $is_airgap ]]; then securityonion_repo >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 fi From 5882642c3255b43714c8195d1947f9cbf1f993ff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 28 Mar 2021 22:10:02 -0400 Subject: [PATCH 026/188] fixpath for GPG Keys for real --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 53901a73a..04fcf529d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -165,9 +165,9 @@ get_random_value() { gpg_rpm_import() { if [[ "$OS" == "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then - local RPMKEYSLOC=$temp_install_dir/salt/common/keys + local RPMKEYSLOC="../salt/common/keys" else - local RPMKEYSLOC=$UPDATEDIR/salt/common/keys + local RPMKEYSLOC="$UPDATEDIR/salt/common/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') From d889bd26946775839a68ec20faed79d15533aaeb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 28 Mar 2021 22:32:03 -0400 Subject: [PATCH 027/188] Fix Security Onio Pub Key --- salt/common/keys/securityonion.pub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/keys/securityonion.pub b/salt/common/keys/securityonion.pub index c9148ff5c..15be14ca9 100644 --- a/salt/common/keys/securityonion.pub +++ b/salt/common/keys/securityonion.pub @@ -49,4 +49,4 @@ cfZ/wnBw1FoNEd51U54euo9yvOgOVtJGvqLgHNwB8574FhQhoWAMhyizqdgeEt26 m3FXecUNKL/AK71/l04vor+/WsXe8uhDg3O84qeYa9wgd8LZZVmGZJDosSwqYjtb LdNNm+v60Zo6rFWSREegqi/nRTTDdxdW99ybjlh+mpbq3xavyFXF =bhkm ------END PGP PUBLIC KEY BLOCK----- \ No newline at end of file +-----END PGP PUBLIC KEY BLOCK----- From 6bce8e8e2cffb4ee655eea8671767cde22c869b4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 29 Mar 2021 07:30:26 -0400 Subject: [PATCH 028/188] Remove incompatible example --- salt/soc/files/soc/custom.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/soc/files/soc/custom.js b/salt/soc/files/soc/custom.js index b23b7c36b..575e019a7 100644 --- a/salt/soc/files/soc/custom.js +++ b/salt/soc/files/soc/custom.js @@ -17,8 +17,5 @@ suggested to avoid and/or minimize the extent of any content placed here so that upgrading to newer version of Security Onion do not become a burden. - - Example: - - i18n.translations["en-US"].loginHeader = "Unauthorized use of this computer system is prohibited..."; + */ From 2ff790699fc9960c85d1566a3558f9ca5840e87c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 29 Mar 2021 09:36:24 -0400 Subject: [PATCH 029/188] [fix] Set ntp_string to empty, not ntp_servers --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 6dd10096b..533a77a92 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -489,7 +489,7 @@ collect_node_ls_pipeline_worker_count() { collect_ntp_servers() { if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then if whiptail_ntp_ask; then - [[ $is_airgap ]] && ntp_servers=() + [[ $is_airgap ]] && ntp_string="" whiptail_ntp_servers "$ntp_string" while ! valid_ntp_list "$ntp_string"; do From 1a58479f39178207ce08c7b028e5af7de2e7c9a8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Mar 2021 15:15:34 -0400 Subject: [PATCH 030/188] Fix acng passthrough --- salt/manager/files/acng/acng.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index 1cc6bf6d9..3492cf111 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -80,7 +80,7 @@ RedirMax: 6 VfileUseRangeOps: 0 # PassThroughPattern: private-ppa\.launchpad\.net:443$ # PassThroughPattern: .* # this would allow CONNECT to everything -PassThroughPattern: (download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ +PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ # ResponseFreezeDetectTime: 500 # ReuseConnections: 1 # PipelineDepth: 255 From f73bf947bcbd662d8693296b3642bdb2764722a6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Mar 2021 15:42:26 -0400 Subject: [PATCH 031/188] Fix repo url --- setup/yum_repos/securityonioncache.repo | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo index a38bac944..a55ee47d4 100644 --- a/setup/yum_repos/securityonioncache.repo +++ b/setup/yum_repos/securityonioncache.repo @@ -1,6 +1,6 @@ [base] name=CentOS-$releasever - Base -baseurl=https://repocache.securityonion.net/file/securityonion-repo/base/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/base/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 @@ -45,11 +45,11 @@ name=SaltStack repo for RHEL/CentOS $releasever PY3 baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ enabled=1 gpgcheck=1 -gpgkey=https://repocache.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub [wazuh_repo] gpgcheck=1 -gpgkey=http://repocache.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ From 3fce63e0c5e4a6142a5b998c29e03092c4119644 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Mar 2021 16:43:44 -0400 Subject: [PATCH 032/188] Fix Repo Again --- setup/yum_repos/securityonioncache.repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo index a55ee47d4..4fcb992d5 100644 --- a/setup/yum_repos/securityonioncache.repo +++ b/setup/yum_repos/securityonioncache.repo @@ -52,5 +52,5 @@ gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository -baseurl=https://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ protect=1 \ No newline at end of file From 0e9ffe033d86ce9ab78df1d6c1d157692ef6e40c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 09:30:06 -0400 Subject: [PATCH 033/188] Show message about setting up network earlier during setup --- setup/so-setup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d9e64105a..982195703 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -298,6 +298,10 @@ if ! [[ -f $install_opt_file ]]; then source "$net_init_file" fi + if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then + whiptail_management_interface_setup + fi + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init fi @@ -315,10 +319,6 @@ if ! [[ -f $install_opt_file ]]; then [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi - if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then - whiptail_management_interface_setup - fi - if [[ $is_minion ]]; then add_mngr_ip_to_hosts fi From 25eca39428a585557183535c3424b0cda00d9479 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 09:54:21 -0400 Subject: [PATCH 034/188] Always ask for ntp setup on iso installs, don't ask on network installs --- setup/so-functions | 48 ++++++++++++++++++---------------------------- setup/so-setup | 5 ++--- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 533a77a92..5c69b817a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -487,20 +487,18 @@ collect_node_ls_pipeline_worker_count() { } collect_ntp_servers() { - if [[ $is_airgap || "$NSMSETUP" = 'ADVANCED' || "$MANAGERADV" = 'ADVANCED' || -n $so_proxy ]]; then - if whiptail_ntp_ask; then - [[ $is_airgap ]] && ntp_string="" + if whiptail_ntp_ask; then + [[ $is_airgap ]] && ntp_string="" + whiptail_ntp_servers "$ntp_string" + + while ! valid_ntp_list "$ntp_string"; do + whiptail_invalid_input whiptail_ntp_servers "$ntp_string" + done - while ! valid_ntp_list "$ntp_string"; do - whiptail_invalid_input - whiptail_ntp_servers "$ntp_string" - done - - IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array - else - ntp_servers=() - fi + IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array + else + ntp_servers=() fi } @@ -725,21 +723,19 @@ configure_ntp() { # Install chrony if it isn't already installed if ! command -v chronyc &> /dev/null; then - if [ "$OS" == centos ]; then - yum -y install chrony - else - retry 50 10 "apt-get -y install chrony" || exit 1 - fi + yum -y install chrony fi - [[ -f $chrony_conf ]] && rm -f $chrony_conf + [[ -f $chrony_conf ]] && mv $chrony_conf "$chrony_conf.bak" + + echo "# Config created by Security Onion" > $chrony_conf # Build list of servers for addr in "${ntp_servers[@]}"; do echo "server $addr iburst" >> $chrony_conf done - printf '%s\n' \ + printf '%s\n\n' \ 'driftfile /var/lib/chrony/drift' \ 'makestep 1.0 3' \ 'rtcsync' \ @@ -748,10 +744,10 @@ configure_ntp() { systemctl enable chronyd systemctl start chronyd - # Sync time & update the system time - chronyc -a 'burst 4/4' - sleep 20 # Wait for chrony to sync - chronyc -a makestep + # Tell the chrony daemon to sync time & update the system time + # Since these commands only make a call to chronyd, wait after each command to make sure the changes are made + chronyc -a 'burst 4/4' && sleep 30 + chronyc -a makestep && sleep 30 } checkin_at_boot() { @@ -761,12 +757,6 @@ checkin_at_boot() { echo "startup_states: highstate" >> "$minion_config" } -check_ntp_configured() { - if systemctl is-active --quiet chronyd || systemctl is-active --quiet ntpd; then - ntp_configured=true - fi -} - check_requirements() { local standalone_or_dist=$1 local node_type=$2 # optional diff --git a/setup/so-setup b/setup/so-setup index 982195703..37121c4fb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -534,8 +534,7 @@ if [[ $is_sensor && ! $is_eval ]]; then fi fi -check_ntp_configured -[[ -z $ntp_configured ]] && collect_ntp_servers +[[ $is_iso ]] && collect_ntp_servers if [[ $is_node && ! $is_eval ]]; then whiptail_node_advanced @@ -584,7 +583,7 @@ set_redirect >> $setup_log 2>&1 # Show initial progress message set_progress_str 0 'Running initial configuration steps' - [[ -z $ntp_configured ]] && [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 + [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 reserve_ports From 5f5a53b8bb9dfb85e23f13fce913b4ea4fbe404b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Mar 2021 11:14:58 -0400 Subject: [PATCH 035/188] Push repolist to dev null --- salt/common/init.sls | 5 ----- setup/so-functions | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 157f2d49a..3e6774219 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -72,11 +72,6 @@ repair_yumdb: - onlyif: - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' -epel: - pkg.installed: - - skip_suggestions: True - - pkgs: - - epel-release {% endif %} # Install common packages diff --git a/setup/so-functions b/setup/so-functions index 99819e9a9..9adb22b86 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2644,6 +2644,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then + yum repolist >> /dev/null yum -y update >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 From 09064baf716afaad69837e64ce16d12434575253 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Mar 2021 11:21:19 -0400 Subject: [PATCH 036/188] Update so-common --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 04fcf529d..ccf211637 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -173,8 +173,8 @@ gpg_rpm_import() { RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') for RPMKEY in "${RPMKEYS[@]}"; do - rpm --import $RPMKEYSLOC/$RPMKEY - echo "Imported $RPMKEY" + rpm --import $RPMKEYSLOC/$RPMKEY + echo "Imported $RPMKEY" done fi } From fc3fd00216913b6446706a185b3dfdeafe95af1a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Mar 2021 11:28:47 -0400 Subject: [PATCH 037/188] Fix formatting --- setup/so-functions | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 9adb22b86..aff7a8375 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1978,19 +1978,6 @@ saltify() { systemctl enable salt-master >> "$setup_log" 2>&1 ;; *) - #if [ "$MANAGERUPDATES" = '1' ]; then - # { - # if [[ ! $is_airgap ]]; then - # Create the GPG Public Key for the Salt Repo - #cp ./public_keys/salt.pem /etc/pki/rpm-gpg/saltstack-signing-key; - - # Copy repo files over - #cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo; - # else - # info "This is airgap" - # fi - # } >> "$setup_log" 2>&1 - #fi ;; esac if [[ ! $is_airgap ]]; then @@ -2645,7 +2632,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then yum repolist >> /dev/null - yum -y update >> "$setup_log" + yum -y update >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 From 7049383ba68d162fe60f0f9391f391eeb1875012 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 30 Mar 2021 15:47:05 +0000 Subject: [PATCH 038/188] Add Elastic scripts --- .../tools/sbin/so-elasticsearch-indices-list | 21 ++++++++++++++++ .../tools/sbin/so-elasticsearch-pipeline-view | 25 +++++++++++++++++++ .../tools/sbin/so-elasticsearch-shards-list | 21 ++++++++++++++++ .../sbin/so-elasticsearch-template-remove | 21 ++++++++++++++++ .../tools/sbin/so-elasticsearch-template-view | 25 +++++++++++++++++++ .../tools/sbin/so-kibana-space-defaults | 0 salt/common/tools/sbin/so-logstash-events | 25 +++++++++++++++++++ .../tools/sbin/so-logstash-pipeline-stats | 25 +++++++++++++++++++ 8 files changed, 163 insertions(+) create mode 100755 salt/common/tools/sbin/so-elasticsearch-indices-list create mode 100755 salt/common/tools/sbin/so-elasticsearch-pipeline-view create mode 100755 salt/common/tools/sbin/so-elasticsearch-shards-list create mode 100755 salt/common/tools/sbin/so-elasticsearch-template-remove create mode 100755 salt/common/tools/sbin/so-elasticsearch-template-view mode change 100644 => 100755 salt/common/tools/sbin/so-kibana-space-defaults create mode 100755 salt/common/tools/sbin/so-logstash-events create mode 100755 salt/common/tools/sbin/so-logstash-pipeline-stats diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-list b/salt/common/tools/sbin/so-elasticsearch-indices-list new file mode 100755 index 000000000..c9df67a25 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-indices-list @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view new file mode 100755 index 000000000..04901e122 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . +else + curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . +fi diff --git a/salt/common/tools/sbin/so-elasticsearch-shards-list b/salt/common/tools/sbin/so-elasticsearch-shards-list new file mode 100755 index 000000000..9d28ed95b --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-shards-list @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +curl -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-template-remove b/salt/common/tools/sbin/so-elasticsearch-template-remove new file mode 100755 index 000000000..f7c3e6812 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-template-remove @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +curl -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 diff --git a/salt/common/tools/sbin/so-elasticsearch-template-view b/salt/common/tools/sbin/so-elasticsearch-template-view new file mode 100755 index 000000000..c9f3ec199 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-template-view @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . +else + curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . +fi diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/common/tools/sbin/so-kibana-space-defaults old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-logstash-events b/salt/common/tools/sbin/so-logstash-events new file mode 100755 index 000000000..817cafb72 --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-events @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done +else + curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events +fi diff --git a/salt/common/tools/sbin/so-logstash-pipeline-stats b/salt/common/tools/sbin/so-logstash-pipeline-stats new file mode 100755 index 000000000..b82a125d2 --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-pipeline-stats @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + +. /usr/sbin/so-common + +if [ "$1" == "" ]; then + curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines +else + curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1 +fi From 679925ebd967160aa3242405127e3ce5b829c07b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 30 Mar 2021 13:29:56 -0400 Subject: [PATCH 039/188] Fix sensor cleanup & playbook sync scripts --- salt/common/tools/sbin/so-playbook-sync | 4 ++++ salt/common/tools/sbin/so-sensor-clean | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-playbook-sync b/salt/common/tools/sbin/so-playbook-sync index 250e4a3ad..a76d398cb 100755 --- a/salt/common/tools/sbin/so-playbook-sync +++ b/salt/common/tools/sbin/so-playbook-sync @@ -17,4 +17,8 @@ . /usr/sbin/so-common +# Check to see if we are already running +IS_RUNNING=$(ps aux | pgrep -f "so-playbook-sync" | wc -l) +[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0 + docker exec so-soctopus python3 playbook_play-sync.py diff --git a/salt/common/tools/sbin/so-sensor-clean b/salt/common/tools/sbin/so-sensor-clean index 63f102f0c..e62c3c4da 100755 --- a/salt/common/tools/sbin/so-sensor-clean +++ b/salt/common/tools/sbin/so-sensor-clean @@ -115,7 +115,7 @@ clean() { } # Check to see if we are already running -IS_RUNNING=$(ps aux | grep "so-sensor-clean" | grep -v grep | wc -l) +IS_RUNNING=$(ps aux | pgrep -f "so-sensor-clean" | wc -l) [ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then From be6eb3ed6c91495a79c8e760f0cf15372a5eee16 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 14:17:05 -0400 Subject: [PATCH 040/188] Restart chrony in case it's already running --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 5c69b817a..87c9b4885 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -742,7 +742,7 @@ configure_ntp() { 'logdir /var/log/chrony' >> $chrony_conf systemctl enable chronyd - systemctl start chronyd + systemctl restart chronyd # Tell the chrony daemon to sync time & update the system time # Since these commands only make a call to chronyd, wait after each command to make sure the changes are made From fd51b327ee1d860221c88dbb30c7c16a9cd105c5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 15:23:57 -0400 Subject: [PATCH 041/188] Add messaging to explain chronyc output to log --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 87c9b4885..d31eb28a3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -746,7 +746,9 @@ configure_ntp() { # Tell the chrony daemon to sync time & update the system time # Since these commands only make a call to chronyd, wait after each command to make sure the changes are made + printf "Syncing chrony time to server: " chronyc -a 'burst 4/4' && sleep 30 + printf "Forcing chrony to update the time: " chronyc -a makestep && sleep 30 } From 177989269fda48a1fa833cbf3e75e2379e07ed1b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Mar 2021 15:50:37 -0400 Subject: [PATCH 042/188] Better formatting of chrony.conf --- setup/so-functions | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d31eb28a3..862854c69 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -728,14 +728,16 @@ configure_ntp() { [[ -f $chrony_conf ]] && mv $chrony_conf "$chrony_conf.bak" - echo "# Config created by Security Onion" > $chrony_conf + printf '%s\n' "# NTP server list" > $chrony_conf # Build list of servers for addr in "${ntp_servers[@]}"; do echo "server $addr iburst" >> $chrony_conf done - printf '%s\n\n' \ + printf '\n%s\n' "# Config options" >> $chrony_conf + + printf '%s\n' \ 'driftfile /var/lib/chrony/drift' \ 'makestep 1.0 3' \ 'rtcsync' \ From 0b9cf57b5f10b73dfffd094e414fca5ebe75042f Mon Sep 17 00:00:00 2001 From: gebhard73 Date: Wed, 31 Mar 2021 14:22:06 +0200 Subject: [PATCH 043/188] Update so-index-list Sort by index name. --- salt/common/tools/sbin/so-index-list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list index dcfebbf58..cf9232150 100755 --- a/salt/common/tools/sbin/so-index-list +++ b/salt/common/tools/sbin/so-index-list @@ -15,4 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -curl -X GET -k -L https://localhost:9200/_cat/indices?v +curl -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" From 942de130caabc46726b7f658de6bf083a53cc60b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 31 Mar 2021 12:24:51 +0000 Subject: [PATCH 044/188] Enforce date type for ingest.timestamp --- salt/elasticsearch/templates/so/so-common-template.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index ebf123fed..c1f0a6755 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -267,9 +267,14 @@ }, "ingest":{ "type":"object", - "dynamic": true + "dynamic": true, + "properties":{ + "timestamp":{ + "type":"date" + } + } }, - "intel":{ + "intel":{ "type":"object", "dynamic": true, "properties":{ From 5203c25971d7bbcdab4fe5b76897ac60b73a0bb4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 09:13:38 -0400 Subject: [PATCH 045/188] Add Wazuh 4 Repo --- setup/yum_repos/securityonion.repo | 8 ++++++++ setup/yum_repos/securityonioncache.repo | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/setup/yum_repos/securityonion.repo b/setup/yum_repos/securityonion.repo index 20c907289..e61829380 100644 --- a/setup/yum_repos/securityonion.repo +++ b/setup/yum_repos/securityonion.repo @@ -53,4 +53,12 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 + +[wazuh4_repo] +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ protect=1 \ No newline at end of file diff --git a/setup/yum_repos/securityonioncache.repo b/setup/yum_repos/securityonioncache.repo index 4fcb992d5..6d5058337 100644 --- a/setup/yum_repos/securityonioncache.repo +++ b/setup/yum_repos/securityonioncache.repo @@ -53,4 +53,12 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=http://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/ +protect=1 + +[wazuh4_repo] +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ protect=1 \ No newline at end of file From c03e2b2c11279bb3fc7b1c53815b01207a252cfa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 09:14:40 -0400 Subject: [PATCH 046/188] Move ntp server array to its own pillar in the minion sls file --- setup/so-functions | 24 +++++++++++++++--------- setup/so-setup | 1 + 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 862854c69..702ccece3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1591,8 +1591,7 @@ manager_pillar() { printf '%s\n'\ " kratoskey: '$KRATOSKEY'"\ "" >> "$pillar_file" - - } +} manager_global() { local global_pillar="$local_salt_dir/pillar/global.sls" @@ -1620,12 +1619,7 @@ manager_global() { " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ - " managerip: '$MAINIP'"\ - " ntp_servers:" > "$global_pillar" - - for addr in "${ntp_servers[@]}"; do - echo " - '$addr'" >> "$global_pillar" - done + " managerip: '$MAINIP'" > "$global_pillar" if [[ $is_airgap ]]; then printf '%s\n'\ @@ -1774,7 +1768,6 @@ manager_global() { " bip: '$DOCKERBIP'"\ "redis_settings:"\ " redis_maxmemory: 812" >> "$global_pillar" - printf '%s\n' '----' >> "$setup_log" 2>&1 } @@ -1837,6 +1830,19 @@ network_setup() { } >> "$setup_log" 2>&1 } +ntp_pillar() { + local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls + + if [[ ${#ntp_servers[@]} -gt 0 ]]; then + printf '%s\n'\ + "ntp:"\ + " servers:" >> "$global_pillar" + for addr in "${ntp_servers[@]}"; do + printf '%s\n' " - '$addr'" >> "$pillar_file" + done + fi +} + parse_install_username() { # parse out the install username so things copy correctly INSTALLUSERNAME=${SUDO_USER:-${USER}} diff --git a/setup/so-setup b/setup/so-setup index 37121c4fb..65be15dc1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -584,6 +584,7 @@ set_redirect >> $setup_log 2>&1 set_progress_str 0 'Running initial configuration steps' [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 + ntp_pillar >> $setup_log 2>&1 reserve_ports From bb39ccc1aa512b72dc400b435030eecdb49356a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 09:25:21 -0400 Subject: [PATCH 047/188] Fix Automation Testing --- setup/automation/eval-net-centos | 2 +- setup/automation/import-net-centos | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/automation/eval-net-centos b/setup/automation/eval-net-centos index abd0c4765..82d2cc9ec 100644 --- a/setup/automation/eval-net-centos +++ b/setup/automation/eval-net-centos @@ -41,7 +41,7 @@ install_type=EVAL # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-net-centos b/setup/automation/import-net-centos index 37ca6ac51..f6394bde1 100644 --- a/setup/automation/import-net-centos +++ b/setup/automation/import-net-centos @@ -41,7 +41,7 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= From 46865809ed92cddb1199ab73a8bad9fd4c3de00c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 09:28:02 -0400 Subject: [PATCH 048/188] Fix Automation Testing round 2 --- setup/automation/import-airgap | 2 +- setup/automation/import-ami | 2 +- setup/automation/import-iso | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/automation/import-airgap b/setup/automation/import-airgap index bfd0e3641..9c394ef2f 100644 --- a/setup/automation/import-airgap +++ b/setup/automation/import-airgap @@ -42,7 +42,7 @@ INTERWEBS=AIRGAP # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-ami b/setup/automation/import-ami index 88734c352..10758be9a 100644 --- a/setup/automation/import-ami +++ b/setup/automation/import-ami @@ -41,7 +41,7 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= diff --git a/setup/automation/import-iso b/setup/automation/import-iso index 011623091..fbfdd364b 100644 --- a/setup/automation/import-iso +++ b/setup/automation/import-iso @@ -41,7 +41,7 @@ install_type=IMPORT # LSPIPELINEBATCH= # LSPIPELINEWORKERS= MANAGERADV=BASIC -MANAGERUPDATES=1 +MANAGERUPDATES=0 # MDNS= # MGATEWAY= # MIP= From 89922a439e279a371973bab7b9ccd71bf2241cc7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 12:37:33 -0400 Subject: [PATCH 049/188] Move repo files --- salt/common/init.sls | 50 ++++++++++++++++++- .../common}/yum_repos/securityonion.repo | 0 .../common}/yum_repos/securityonioncache.repo | 0 setup/so-functions | 4 +- 4 files changed, 51 insertions(+), 3 deletions(-) rename {setup => salt/common}/yum_repos/securityonion.repo (100%) rename {setup => salt/common}/yum_repos/securityonioncache.repo (100%) diff --git a/salt/common/init.sls b/salt/common/init.sls index 3e6774219..d0dae49f5 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -64,7 +64,7 @@ salttmp: - group: 939 - makedirs: True -# Install epel +# Remove default Repos {% if grains['os'] == 'CentOS' %} repair_yumdb: cmd.run: @@ -72,6 +72,54 @@ repair_yumdb: - onlyif: - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' +crbase: + file.absent: + - name: /etc/yum.repos.d/CentOS-Base.repo + +crcr: + file.absent: + - name: /etc/yum.repos.d/CentOS-CR.repo + +crdebug: + file.absent: + - name: /etc/yum.repos.d/CentOS-Debuginfo.repo + +crfasttrack: + file.absent: + - name: /etc/yum.repos.d/CentOS-fasttrack.repo + +crmedia: + file.absent: + - name: /etc/yum.repos.d/CentOS-Media.repo + +crsources: + file.absent: + - name: /etc/yum.repos.d/CentOS-Sources.repo + +crvault: + file.absent: + - name: /etc/yum.repos.d/CentOS-Vault.repo + +crkernel: + file.absent: + - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo + +crepel: + file.absent: + - name: /etc/yum.repos.d/epel.repo + +crtesting: + file.absent: + - name: /etc/yum.repos.d/epel-testing.repo + +crssrepo: + file.absent: + - name: /etc/yum.repos.d/saltstack.repo + +crwazrepo: + file.absent: + - name: /etc/yum.repos.d/wazuh.repo + {% endif %} # Install common packages diff --git a/setup/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo similarity index 100% rename from setup/yum_repos/securityonion.repo rename to salt/common/yum_repos/securityonion.repo diff --git a/setup/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo similarity index 100% rename from setup/yum_repos/securityonioncache.repo rename to salt/common/yum_repos/securityonioncache.repo diff --git a/setup/so-functions b/setup/so-functions index aff7a8375..63cb5ca7c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2215,9 +2215,9 @@ securityonion_repo() { mv /etc/yum.repos.d/* /root/oldrepos/ rm -f /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ./yum_repos/securityonioncache.repo /etc/yum.repos.d/ + cp -f ../../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else - cp -f ./yum_repos/securityonion.repo /etc/yum.repos.d/ + cp -f ../../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ fi else echo "This is Ubuntu" From f8d72413544642bf9feab5abdc8e0a96c3e218e9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 12:55:46 -0400 Subject: [PATCH 050/188] Fix repo file path --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 63cb5ca7c..73293bc47 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2215,9 +2215,9 @@ securityonion_repo() { mv /etc/yum.repos.d/* /root/oldrepos/ rm -f /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else - cp -f ../../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ fi else echo "This is Ubuntu" From 1c4ba28336423c6164c671615e2d90d52e4fd4c4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 13:28:42 -0400 Subject: [PATCH 051/188] [fix] host_pillar overwrites the file, so run ntp_pillar after it --- setup/so-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 84e94e780..e2c866964 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -585,7 +585,6 @@ set_redirect >> $setup_log 2>&1 set_progress_str 0 'Running initial configuration steps' [[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1 - ntp_pillar >> $setup_log 2>&1 reserve_ports @@ -619,6 +618,8 @@ set_redirect >> $setup_log 2>&1 fi host_pillar >> $setup_log 2>&1 + ntp_pillar >> $setup_log 2>&1 + if [[ $is_minion || $is_import ]]; then set_updates >> $setup_log 2>&1 From 761a12ebbb6a95f8414f7b6e07b074a46ae025c2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 13:32:49 -0400 Subject: [PATCH 052/188] Fix variable name --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 6b4f693e3..2732a0ee9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1802,7 +1802,7 @@ ntp_pillar() { if [[ ${#ntp_servers[@]} -gt 0 ]]; then printf '%s\n'\ "ntp:"\ - " servers:" >> "$global_pillar" + " servers:" >> "$pillar_file" for addr in "${ntp_servers[@]}"; do printf '%s\n' " - '$addr'" >> "$pillar_file" done From 820b01405f428307aa23286ed7c9e9ef0c5f8a24 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 31 Mar 2021 14:57:36 -0400 Subject: [PATCH 053/188] For hunt quick actions, pipe value to 'escape' operator to escape backslashes and double quotes --- salt/soc/files/soc/alerts.actions.json | 2 +- salt/soc/files/soc/hunt.actions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/files/soc/alerts.actions.json b/salt/soc/files/soc/alerts.actions.json index 364c59d27..c0543d8fc 100644 --- a/salt/soc/files/soc/alerts.actions.json +++ b/salt/soc/files/soc/alerts.actions.json @@ -1,7 +1,7 @@ [ { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", "links": [ - "/#/hunt?q=\"{value}\" | groupby event.module event.dataset" + "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" ]}, { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", "links": [ diff --git a/salt/soc/files/soc/hunt.actions.json b/salt/soc/files/soc/hunt.actions.json index 364c59d27..c0543d8fc 100644 --- a/salt/soc/files/soc/hunt.actions.json +++ b/salt/soc/files/soc/hunt.actions.json @@ -1,7 +1,7 @@ [ { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", "links": [ - "/#/hunt?q=\"{value}\" | groupby event.module event.dataset" + "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" ]}, { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", "links": [ From f7e99b496134345e81ef8f787bb627997773358d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 31 Mar 2021 15:17:15 -0400 Subject: [PATCH 054/188] https://github.com/Security-Onion-Solutions/securityonion/issues/3709 --- salt/firewall/map.jinja | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index 2df668a07..496e6f568 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -18,14 +18,18 @@ {# This block translate the portgroups defined in the pillar to what is defined my portgroups.yaml and portgroups.local.yaml #} {% if salt['pillar.get']('firewall:assigned_hostgroups:chain') %} + {% set translated_pillar_assigned_hostgroups = {'chain': {}} %} {% for chain, hg in salt['pillar.get']('firewall:assigned_hostgroups:chain').items() %} {% for pillar_hostgroup, pillar_portgroups in salt['pillar.get']('firewall:assigned_hostgroups:chain')[chain].hostgroups.items() %} - {% do translated_pillar_assigned_hostgroups.update({"chain": {chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}}) %} + {% if translated_pillar_assigned_hostgroups.chain[chain] is defined %} + {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups.update({pillar_hostgroup: {"portgroups": []}}) %} + {% else %} + {% do translated_pillar_assigned_hostgroups.chain.update({chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}) %} + {% endif %} {% for pillar_portgroup in pillar_portgroups.portgroups %} {% set pillar_portgroup = pillar_portgroup.split('.') | last %} {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups[pillar_hostgroup].portgroups.append(defined_portgroups[pillar_portgroup]) %} - {% endfor %} {% endfor %} {% endfor %} @@ -39,7 +43,6 @@ {% set assigned_hostgroups = default_assigned_hostgroups.role[role] %} {% endif %} - {% if translated_pillar_assigned_hostgroups %} {% do salt['defaults.merge'](assigned_hostgroups, translated_pillar_assigned_hostgroups, merge_lists=True, in_place=True) %} {% endif %} \ No newline at end of file From ef984455605de8b1cb6f1ea7ee7b13d49afa824a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 31 Mar 2021 15:44:41 -0400 Subject: [PATCH 055/188] Fix Playbook Alert timestamps --- salt/elastalert/files/modules/so/playbook-es.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index c10a80f2c..ab2327ab7 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -17,7 +17,7 @@ class PlaybookESAlerter(Alerter): def alert(self, matches): for match in matches: today = strftime("%Y.%m.%d", gmtime()) - timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime()) + timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime()) headers = {"Content-Type": "application/json"} payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" From 3de980e4a1d0b9fd41aab9e87edf1293d5ba7fb6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 31 Mar 2021 16:00:37 -0400 Subject: [PATCH 056/188] Move function call to run after Network Manager is installed --- setup/so-setup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index e2c866964..4a19b1b50 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -168,10 +168,8 @@ set_ssh_cmds $automated local_sbin="$(pwd)/../salt/common/tools/sbin" export PATH=$PATH:$local_sbin -set_network_dev_status_list set_palette >> $setup_log 2>&1 - # Kernel messages can overwrite whiptail screen #812 # https://github.com/Security-Onion-Solutions/securityonion/issues/812 dmesg -D @@ -434,6 +432,7 @@ if [[ $is_helix ]]; then fi if [[ $is_helix || $is_sensor ]]; then + set_network_dev_status_list whiptail_sensor_nics fi From 9b84a92ced276c4dc045e67cb501d89860c4a20f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:47:04 -0400 Subject: [PATCH 057/188] Manage the repo files --- salt/common/init.sls | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index d0dae49f5..1ee64bb5a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -2,6 +2,7 @@ {% if sls in allowed_states %} {% set role = grains.id.split('_') | last %} +{% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -84,6 +85,10 @@ crdebug: file.absent: - name: /etc/yum.repos.d/CentOS-Debuginfo.repo +crdockerce: + file.absent: + - name: /etc/yum.repos.d/docker-ce.repo + crfasttrack: file.absent: - name: /etc/yum.repos.d/CentOS-fasttrack.repo @@ -120,6 +125,17 @@ crwazrepo: file.absent: - name: /etc/yum.repos.d/wazuh.repo +crsecurityonionrepo: + file.managed: + {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] or managerupdates == 0 %} + - name: /etc/yum.repos.d/securityonion.repo + - source: salt://common/yum_repos.d/securityonion.repo + {% else %} + - name: /etc/yum.repos.d/securityonioncache.repo + - source: salt://commmon/yum_repos/securityonioncache.repo + {% endif %} + - mode: 644 + {% endif %} # Install common packages From 358f39753556883ba11034b3072dd970aa7a747f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:50:43 -0400 Subject: [PATCH 058/188] Manage the repo files --- salt/common/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 1ee64bb5a..c2ccb908f 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -88,7 +88,7 @@ crdebug: crdockerce: file.absent: - name: /etc/yum.repos.d/docker-ce.repo - + crfasttrack: file.absent: - name: /etc/yum.repos.d/CentOS-fasttrack.repo @@ -129,7 +129,7 @@ crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] or managerupdates == 0 %} - name: /etc/yum.repos.d/securityonion.repo - - source: salt://common/yum_repos.d/securityonion.repo + - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - source: salt://commmon/yum_repos/securityonioncache.repo From f387c4327a024f2f6993d5185cb5fe27628d51b3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:53:20 -0400 Subject: [PATCH 059/188] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index c2ccb908f..f15dfe70d 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -127,7 +127,7 @@ crwazrepo: crsecurityonionrepo: file.managed: - {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] or managerupdates == 0 %} + {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] %} - name: /etc/yum.repos.d/securityonion.repo - source: salt://common/yum_repos/securityonion.repo {% else %} From 13421bb04bb0b17f351c3f187382575c8c74a1dc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 16:59:15 -0400 Subject: [PATCH 060/188] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index f15dfe70d..0571cf2b3 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -132,7 +132,7 @@ crsecurityonionrepo: - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - - source: salt://commmon/yum_repos/securityonioncache.repo + - source: salt://common/yum_repos/securityonioncache.repo {% endif %} - mode: 644 From bfc5bb011f0d137a85dff07b64478cb8e3976219 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:03:52 -0400 Subject: [PATCH 061/188] Manage the repo files --- salt/common/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 0571cf2b3..5d95e69b6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -88,7 +88,7 @@ crdebug: crdockerce: file.absent: - name: /etc/yum.repos.d/docker-ce.repo - + crfasttrack: file.absent: - name: /etc/yum.repos.d/CentOS-fasttrack.repo @@ -127,9 +127,9 @@ crwazrepo: crsecurityonionrepo: file.managed: - {% if role in ['eval', 'standalone', 'import', 'manager' 'managersearch'] %} + {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - name: /etc/yum.repos.d/securityonion.repo - - source: salt://common/yum_repos/securityonion.repo + - source: salt://common/yum_repos.d/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - source: salt://common/yum_repos/securityonioncache.repo From 1509722185ebb4a8432edfcc0fe3dfc1fd27da4c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:04:56 -0400 Subject: [PATCH 062/188] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 5d95e69b6..f27af7233 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -128,7 +128,7 @@ crwazrepo: crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - - name: /etc/yum.repos.d/securityonion.repo + - name: /etc/yum.repos/securityonion.repo - source: salt://common/yum_repos.d/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo From 96459885556393912a373c49d72c9e2bc7714bf3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:06:26 -0400 Subject: [PATCH 063/188] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index f27af7233..2bfc6b66c 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -129,7 +129,7 @@ crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - name: /etc/yum.repos/securityonion.repo - - source: salt://common/yum_repos.d/securityonion.repo + - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo - source: salt://common/yum_repos/securityonioncache.repo From 88eab865286fcd394a39b589868613c19f82dfc9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Mar 2021 17:07:30 -0400 Subject: [PATCH 064/188] Manage the repo files --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 2bfc6b66c..44a00024a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -128,7 +128,7 @@ crwazrepo: crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - - name: /etc/yum.repos/securityonion.repo + - name: /etc/yum.repos.d/securityonion.repo - source: salt://common/yum_repos/securityonion.repo {% else %} - name: /etc/yum.repos.d/securityonioncache.repo From bc04cae91896c986575a44bcc0a812cb3d575d24 Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Thu, 1 Apr 2021 16:59:47 +0900 Subject: [PATCH 065/188] Fix: Connection to ES is "https" from 2.3.40 --- salt/curator/files/bin/so-curator-closed-delete-delete | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 58433ee1a..9cc94833c 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -34,7 +34,7 @@ overlimit() { closedindices() { - INDICES=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null) + INDICES=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null) [ $? -eq 1 ] && return false echo ${INDICES} | grep -q -E "(logstash-|so-)" } @@ -49,12 +49,12 @@ while overlimit && closedindices; do # First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed. # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. # Finally, select the first entry in that sorted list. - OLDEST_INDEX=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) + OLDEST_INDEX=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. - curl -XDELETE -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} + curl -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} # Finally, write a log entry that says we deleted it. echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG} -done \ No newline at end of file +done From 0d056123934fb754469191ac3ceb6e63abc04e40 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Apr 2021 10:00:55 -0400 Subject: [PATCH 066/188] Reserve ports for Zeek --- salt/common/files/99-reserved-ports.conf | 2 +- salt/common/init.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/files/99-reserved-ports.conf b/salt/common/files/99-reserved-ports.conf index a846341a5..208ef0acc 100644 --- a/salt/common/files/99-reserved-ports.conf +++ b/salt/common/files/99-reserved-ports.conf @@ -1 +1 @@ -net.ipv4.ip_local_reserved_ports=55000,57314 +net.ipv4.ip_local_reserved_ports=55000,57314,47760,47761,47762 diff --git a/salt/common/init.sls b/salt/common/init.sls index 3e6774219..6d0e567c5 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -268,7 +268,7 @@ docker: # Reserve OS ports for Docker proxy in case boot settings are not already applied/present dockerapplyports: cmd.run: - - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314"; fi + - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760,47761,47762"; fi # Reserve OS ports for Docker proxy dockerreserveports: From 40313fc2f5fd9b087f89bb1adda7f4ce0269da52 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Apr 2021 10:29:58 -0400 Subject: [PATCH 067/188] Reserve ports for Zeek --- salt/common/files/99-reserved-ports.conf | 2 +- salt/common/init.sls | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/files/99-reserved-ports.conf b/salt/common/files/99-reserved-ports.conf index 208ef0acc..ac4391693 100644 --- a/salt/common/files/99-reserved-ports.conf +++ b/salt/common/files/99-reserved-ports.conf @@ -1 +1 @@ -net.ipv4.ip_local_reserved_ports=55000,57314,47760,47761,47762 +net.ipv4.ip_local_reserved_ports=55000,57314,55000,57314,47760-47860 \ No newline at end of file diff --git a/salt/common/init.sls b/salt/common/init.sls index 6d0e567c5..7945a678a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -266,9 +266,10 @@ docker: - file: docker_daemon # Reserve OS ports for Docker proxy in case boot settings are not already applied/present +# 55000 = Wazuh, 57314 = Strelka, 47760-47860 = Zeek dockerapplyports: cmd.run: - - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760,47761,47762"; fi + - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760-47860"; fi # Reserve OS ports for Docker proxy dockerreserveports: From 7c6b037ae55ef36727e49b4b3786cc3fd8eff57c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Apr 2021 10:30:52 -0400 Subject: [PATCH 068/188] Reserve ports for Zeek --- salt/common/files/99-reserved-ports.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/files/99-reserved-ports.conf b/salt/common/files/99-reserved-ports.conf index ac4391693..82eb03f79 100644 --- a/salt/common/files/99-reserved-ports.conf +++ b/salt/common/files/99-reserved-ports.conf @@ -1 +1 @@ -net.ipv4.ip_local_reserved_ports=55000,57314,55000,57314,47760-47860 \ No newline at end of file +net.ipv4.ip_local_reserved_ports=55000,57314,47760-47860 \ No newline at end of file From 43c31b4e665fc993f6d5f6d07506ef19619cc924 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 1 Apr 2021 14:56:05 -0400 Subject: [PATCH 069/188] Fix script so changes are actually made --- salt/common/tools/sbin/so-ssh-harden | 70 ++++++++++++---------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 0e0b28b39..532e2f737 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -4,6 +4,8 @@ if [[ $1 =~ ^(-q|--quiet) ]]; then quiet=true +elif [[ $1 =~ ^(-v|--verbose) ]]; then + verbose=true fi sshd_config=/etc/ssh/sshd_config @@ -12,39 +14,27 @@ temp_config=/tmp/sshd_config before= after= reload_required=false +change_header_printed=false check_sshd_t() { local string=$1 - local state=$2 local grep_out grep_out=$(sshd -T | grep "^${string}") - if [[ $state == "Before" ]]; then - before=$grep_out - else - after=$grep_out - fi + before=$grep_out } print_diff() { - local type=$1 local diff - diff=$(diff -dqbB <(echo $before) <(echo $after)) + diff=$(diff -dbB <(echo $before) <(echo $after) | awk 'NR>1') if [[ -n $diff ]]; then - printf '%s\n' "$type" "$diff" - echo "" - fi -} - -print_msg() { - local msg=$1 - if ! [[ $quiet ]]; then - printf "%s\n" \ - "----" \ - "$msg" \ - "----" + if [[ $change_header_printed == false ]]; then + printf '%s\n' '' "Changes" '-------' '' + change_header_printed=true + fi + echo -e "$diff\n" fi } @@ -77,65 +67,65 @@ main() { echo "" >> $temp_config # Ciphers - check_sshd_t "ciphers" "Before" + check_sshd_t "ciphers" local cipher_string cipher_string=$(echo "$before" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g") - check_sshd_t "ciphers" "After" + after=$cipher_string - if ! [[ $quiet ]]; then print_diff "ciphers"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then + if [[ $before != "$after" ]]; then add_if_missing "$cipher_string" && test_config || exit 1 fi # KexAlgorithms - check_sshd_t "kexalgorithms" "Before" + check_sshd_t "kexalgorithms" local kexalg_string kexalg_string=$(echo "$before" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g") - check_sshd_t "kexalgorithms" "After" + after=$kexalg_string - if ! [[ $quiet ]]; then print_diff "kexalgorithms"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then + if [[ $before != "$after" ]]; then add_if_missing "$kexalg_string" && test_config || exit 1 fi # Macs - check_sshd_t "macs" "Before" + check_sshd_t "macs" local macs_string macs_string=$(echo "$before" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g") - check_sshd_t "macs" "After" + after=$macs_string - if ! [[ $quiet ]]; then print_diff "macs"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then - add_if_missing "$mac_string" && test_config || exit 1 + if [[ $before != "$after" ]]; then + add_if_missing "$macs_string" && test_config || exit 1 fi # HostKeyAlgorithms - check_sshd_t "hostkeyalgorithms" "Before" + check_sshd_t "hostkeyalgorithms" local hostkeyalg_string hostkeyalg_string=$(echo "$before" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g") - check_sshd_t "hostkeyalgorithms" "After" + after=$hostkeyalg_string - if ! [[ $quiet ]]; then print_diff "hostkeyalgorithms"; fi + if [[ $verbose ]]; then print_diff; fi - if [[ $before != $after ]]; then + if [[ $before != "$after" ]]; then add_if_missing "$hostkeyalg_string" && test_config || exit 1 fi if [[ $reload_required == true ]]; then mv -f $temp_config $sshd_config - if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes..."; fi + if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes"; fi systemctl reload sshd - print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." + echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." else - if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up."; fi + if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up"; fi rm -f $temp_config fi } From fd57996bc6980ce7ddb7147f8bd43039337311e8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 2 Apr 2021 10:00:27 -0400 Subject: [PATCH 070/188] Change behavior of adding lines to sshd config * Replace existing lines in cases where a change has already been made --- salt/common/tools/sbin/so-ssh-harden | 20 +++++++++++--------- setup/so-setup | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 532e2f737..bdec9894b 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -38,12 +38,14 @@ print_diff() { fi } -add_if_missing() { - local string=$1 - if ! grep -q "$1" $temp_config; then - printf "%s\n\n" "$1" >> $temp_config - reload_required=true +replace_or_add() { + local type=$1 + local string=$2 + if grep -q "$type" $temp_config; then + sed -i "/$type .*/d" $temp_config fi + printf "%s\n\n" "$string" >> $temp_config + reload_required=true } test_config() { @@ -76,7 +78,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$cipher_string" && test_config || exit 1 + replace_or_add "ciphers" "$cipher_string" && test_config || exit 1 fi # KexAlgorithms @@ -90,7 +92,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$kexalg_string" && test_config || exit 1 + replace_or_add "kexalgorithms" "$kexalg_string" && test_config || exit 1 fi # Macs @@ -103,7 +105,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$macs_string" && test_config || exit 1 + replace_or_add "macs" "$macs_string" && test_config || exit 1 fi # HostKeyAlgorithms @@ -116,7 +118,7 @@ main() { if [[ $verbose ]]; then print_diff; fi if [[ $before != "$after" ]]; then - add_if_missing "$hostkeyalg_string" && test_config || exit 1 + replace_or_add "hostkeyalgorithms" "$hostkeyalg_string" && test_config || exit 1 fi if [[ $reload_required == true ]]; then diff --git a/setup/so-setup b/setup/so-setup index a742630f7..584dc7933 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -906,7 +906,7 @@ set_redirect >> $setup_log 2>&1 set_progress_str 85 'Applying finishing touches' filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 - so-ssh-harden -q >> $setup_log 2>&1 + so-ssh-harden >> $setup_log 2>&1 if [[ $is_manager || $is_import ]]; then set_progress_str 87 'Adding user to SOC' From 8b8086b91a16b9e2a382d139f17db7651e9ddbdc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 2 Apr 2021 10:20:28 -0400 Subject: [PATCH 071/188] Update wording, as the new key tends to be ED25519, not ECDSA --- salt/common/tools/sbin/so-ssh-harden | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index bdec9894b..5e740c4dd 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -125,7 +125,7 @@ main() { mv -f $temp_config $sshd_config if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes"; fi systemctl reload sshd - echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting." + echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the host key fingerprint for this server before reconnecting." else if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up"; fi rm -f $temp_config From d19c03efef69eaf4b9ddad6dea271764fad038bc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 2 Apr 2021 14:49:22 -0400 Subject: [PATCH 072/188] Refactor search of config lines * Create arrays for each line and loop through them for better code readability * Add more host key algorithms for removal * Update regex to look for a comma or EOL at the end of the search term, to avoid missing last item in list --- salt/common/tools/sbin/so-ssh-harden | 68 ++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-ssh-harden b/salt/common/tools/sbin/so-ssh-harden index 5e740c4dd..2a057ff5e 100755 --- a/salt/common/tools/sbin/so-ssh-harden +++ b/salt/common/tools/sbin/so-ssh-harden @@ -70,8 +70,23 @@ main() { # Ciphers check_sshd_t "ciphers" - local cipher_string - cipher_string=$(echo "$before" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g") + + local bad_ciphers=( + "3des-cbc" + "aes128-cbc" + "aes192-cbc" + "aes256-cbc" + "arcfour" + "arcfour128" + "arcfour256" + "blowfish-cbc" + "cast128-cbc" + ) + + local cipher_string=$before + for cipher in "${bad_ciphers[@]}"; do + cipher_string=$(echo "$cipher_string" | sed "s/${cipher}\(,\|\$\)//g" | sed 's/,$//') + done after=$cipher_string @@ -84,8 +99,20 @@ main() { # KexAlgorithms check_sshd_t "kexalgorithms" - local kexalg_string - kexalg_string=$(echo "$before" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g") + local bad_kexalgs=( + "diffie-hellman-group-exchange-sha1" + "diffie-hellman-group-exchange-sha256" + "diffie-hellman-group1-sha1" + "diffie-hellman-group14-sha1" + "ecdh-sha2-nistp256" + "ecdh-sha2-nistp521" + "ecdh-sha2-nistp384" + ) + + local kexalg_string=$before + for kexalg in "${bad_kexalgs[@]}"; do + kexalg_string=$(echo "$kexalg_string" | sed "s/${kexalg}\(,\|\$\)//g" | sed 's/,$//') + done after=$kexalg_string @@ -97,8 +124,21 @@ main() { # Macs check_sshd_t "macs" - local macs_string - macs_string=$(echo "$before" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g") + + local bad_macs=( + "hmac-sha2-512" + "umac-128@openssh.com" + "hmac-sha2-256" + "umac-64@openssh.com" + "hmac-sha1" + "hmac-sha1-etm@openssh.com" + "umac-64-etm@openssh.com" + ) + + local macs_string=$before + for mac in "${bad_macs[@]}"; do + macs_string=$(echo "$macs_string" | sed "s/${mac}\(,\|\$\)//g" | sed 's/,$//') + done after=$macs_string @@ -110,8 +150,20 @@ main() { # HostKeyAlgorithms check_sshd_t "hostkeyalgorithms" - local hostkeyalg_string - hostkeyalg_string=$(echo "$before" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g") + + local optional_suffix_regex_hka="\(-cert-v01@openssh.com\)\?" + local bad_hostkeyalg_list=( + "ecdsa-sha2-nistp256" + "ecdsa-sha2-nistp384" + "ecdsa-sha2-nistp521" + "ssh-rsa" + "ssh-dss" + ) + + local hostkeyalg_string=$before + for alg in "${bad_hostkeyalg_list[@]}"; do + hostkeyalg_string=$(echo "$hostkeyalg_string" | sed "s/${alg}${optional_suffix_regex_hka}\(,\|\$\)//g" | sed 's/,$//') + done after=$hostkeyalg_string From e430be1017b5a46e11b43bbb6bc97ebb7f398f67 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 2 Apr 2021 16:36:29 -0400 Subject: [PATCH 073/188] Enable Flux compatibility mode to prepare for eventual migration to 2.0 --- salt/influxdb/etc/influxdb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/etc/influxdb.conf b/salt/influxdb/etc/influxdb.conf index 86c1ccfe8..9d89ca774 100644 --- a/salt/influxdb/etc/influxdb.conf +++ b/salt/influxdb/etc/influxdb.conf @@ -233,7 +233,7 @@ # enabled = true # Determines whether the Flux query endpoint is enabled. - # flux-enabled = false + flux-enabled = true # The bind address used by the HTTP service. # bind-address = ":8086" From 8ca0626387b6c8f61a4e79b92180c2e816cc8f2d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 5 Apr 2021 06:55:40 -0400 Subject: [PATCH 074/188] FIX: Hunt query for HTTP EXE downloads should work for both Zeek and Suricata #3753 --- salt/soc/files/soc/hunt.queries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index 840b4b373..93295364d 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -34,7 +34,7 @@ { "name": "HTTP", "description": "HTTP grouped by status code and message", "query": "event.dataset:http | groupby http.status_code http.status_message"}, { "name": "HTTP", "description": "HTTP grouped by method and user agent", "query": "event.dataset:http | groupby http.method http.useragent"}, { "name": "HTTP", "description": "HTTP grouped by virtual host", "query": "event.dataset:http | groupby http.virtual_host"}, - { "name": "HTTP", "description": "HTTP with exe downloads", "query": "event.dataset:http AND file.resp_mime_types:dosexec | groupby http.virtual_host"}, + { "name": "HTTP", "description": "HTTP with exe downloads", "query": "event.dataset:http AND (file.resp_mime_types:dosexec OR file.resp_mime_types:executable) | groupby http.virtual_host"}, { "name": "Intel", "description": "Intel framework hits grouped by indicator", "query": "event.dataset:intel | groupby intel.indicator.keyword"}, { "name": "IRC", "description": "IRC grouped by command", "query": "event.dataset:irc | groupby irc.command.type"}, { "name": "KERBEROS", "description": "KERBEROS grouped by service", "query": "event.dataset:kerberos | groupby kerberos.service"}, From 534dbf9761ecd7680dfd0ebbe9eb8b77475edccc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 09:07:00 -0400 Subject: [PATCH 075/188] change the upgrade command - https://github.com/Security-Onion-Solutions/securityonion/issues/3501 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 7ef63bd68..80646f4ca 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -19,12 +19,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum clean all ; yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" ; /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common ; apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; apt-mark hold salt-common && apt-mark hold salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %} From 1c3a7094bdc8607c74315ace5a8a7e70c83b99f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 11:05:48 -0400 Subject: [PATCH 076/188] upgrade salt to 3003.1 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- setup/so-functions | 18 +++++++++--------- setup/so-preflight | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 2b50b517b..e97115ec1 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3002.5 \ No newline at end of file + version: 3003.1 \ No newline at end of file diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index e6b1303ed..3993c827b 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,5 +2,5 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3002.5 + version: 3003.1 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 2732a0ee9..75e8951ad 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2029,13 +2029,13 @@ saltify() { # Download Ubuntu Keys in case manager updates = 1 mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 if [[ ! $is_airgap ]]; then - logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" + logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" logCmd "wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH" fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then - logCmd "yum -y install salt-master-3002.5" + logCmd "yum -y install salt-master-3003.1" fi systemctl enable salt-master >> "$setup_log" 2>&1 ;; @@ -2048,7 +2048,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - yum -y install salt-minion-3002.5\ + yum -y install salt-minion-3003.1\ python3\ python36-docker\ python36-dateutil\ @@ -2100,8 +2100,8 @@ saltify() { 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR') # Add saltstack repo(s) - wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 - echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" + wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 + echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" # Add Docker repo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - >> "$setup_log" 2>&1 @@ -2109,7 +2109,7 @@ saltify() { # Get gpg keys mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 - wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com$py_ver_url_path/ubuntu/"$ubuntu_version"/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 + wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com$py_ver_url_path/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg >> "$setup_log" 2>&1 wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH >> "$setup_log" 2>&1 @@ -2122,7 +2122,7 @@ saltify() { set_progress_str 6 'Installing various dependencies' retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1 set_progress_str 7 'Installing salt-master' - retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-master=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 ;; *) @@ -2133,14 +2133,14 @@ saltify() { echo "Using apt-key add to add SALTSTACK-GPG-KEY.pub and GPG-KEY-WAZUH" >> "$setup_log" 2>&1 apt-key add "$temp_install_dir"/gpg/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 apt-key add "$temp_install_dir"/gpg/GPG-KEY-WAZUH >> "$setup_log" 2>&1 - echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" + echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" ;; esac retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 set_progress_str 8 'Installing salt-minion & python modules' - retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-minion=3003.1+ds-1 salt-common=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1 diff --git a/setup/so-preflight b/setup/so-preflight index da25e6775..1ef840284 100644 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -46,8 +46,8 @@ check_new_repos() { if [[ $OS == 'centos' ]]; then local repo_arr=( "https://download.docker.com/linux/centos/docker-ce.repo" - "https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/SALTSTACK-GPG-KEY.pub" - "https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3003/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" "https://download.docker.com/linux/ubuntu/gpg" "https://packages.wazuh.com/key/GPG-KEY-WAZUH" "https://packages.wazuh.com/3.x/yum/" @@ -59,7 +59,7 @@ check_new_repos() { local repo_arr=( "https://download.docker.com/linux/ubuntu/gpg" "https://download.docker.com/linux/ubuntu" - "https://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" "https://packages.wazuh.com/key/GPG-KEY-WAZUH" "https://packages.wazuh.com" ) From f9dc040c7fc35fded3986e7c7e0947ae365411cb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 11:38:39 -0400 Subject: [PATCH 077/188] Fix Raid --- salt/common/tools/sbin/so-raid-status | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index d55d158fe..11909e012 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -66,11 +66,13 @@ mkdir -p /opt/so/log/raid {%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %} #check_boss_raid check_software_raid -echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log +#echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log +echo "osraid=1 nsmraid=$SWRAID" > /opt/so/log/raid/status.log {%- elif grains['sosmodel'] in ['SOS1000F', 'SOS1000', 'SOSSN7200', 'SOS10K', 'SOS4000'] %} #check_boss_raid check_lsi_raid -echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log +#echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log +echo "osraid=1 nsmraid=$LSIRAID" > /opt/so/log/raid/status.log {%- else %} exit 0 {%- endif %} From 5aefa2a02408f7f128831e34776d6430f22e1f80 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 11:41:19 -0400 Subject: [PATCH 078/188] Fix Raid for Jertel compliance --- salt/telegraf/scripts/raid.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index c53644889..0938bb658 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -27,7 +27,7 @@ RAIDLOG=/var/log/raid/status.log RAIDSTATUS=$(cat /var/log/raid/status.log) if [ -f "$RAIDLOG" ]; then - echo "raid raidstatus=$RAIDSTATUS " + echo "raid $RAIDSTATUS" else exit 0 fi From bad22ab541ba746bcf4b15a1f727d98d6ee63758 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 12:08:38 -0400 Subject: [PATCH 079/188] Add model to sensoroni config --- salt/sensoroni/files/sensoroni.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 23b967b04..dead05b64 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,5 +1,9 @@ {%- set URLBASE = salt['pillar.get']('global:url_base') %} -{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %} +{%- if salt['pillar.get']('sensoroni:node_description') %} +{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %} +{%- else %} +{%- set DESCRIPTION = salt['grains.get']('sosmodel', '') %} +{%- endif %} {%- set ADDRESS = salt['pillar.get']('sensoroni:node_address') %} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} {%- set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) %} From a824813cdb95b3a6b86f57026d9729b0e807bb00 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 12:10:29 -0400 Subject: [PATCH 080/188] Add model to sensoroni config --- salt/sensoroni/files/sensoroni.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index dead05b64..df2990404 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -13,7 +13,7 @@ {%- else %} {%- set STENODEFAULT = False %} {%- endif %} -{%- set STENOENABLED = salt['pillar.get']('steno:enabled', STENODEFAULT) %} +{%- set STENOENABLED = salt['pillar.get']('steno:enabled', STENODEFAULT) %} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", From 9b8b5e6173b8b3b105dd570edc6c956c0bf08113 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 14:12:24 -0400 Subject: [PATCH 081/188] use -r by default to disable salt bootstrap from doing repo things --- salt/salt/map.jinja | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 80646f4ca..6b5273b84 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -18,11 +18,7 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} - {% if ISAIRGAP is sameas true %} {% set UPGRADECOMMAND = 'yum clean all ; yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} - {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" ; /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} - {% endif %} {% elif grains.os|lower == 'ubuntu' %} {% set UPGRADECOMMAND = 'apt-mark unhold salt-common ; apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; apt-mark hold salt-common && apt-mark hold salt-minion' %} {% endif %} From fc886341591d6d34cbdf1d4057f63aada1363ceb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 15:01:21 -0400 Subject: [PATCH 082/188] Set the Repo for airgap during install --- setup/so-functions | 11 +++++++++++ setup/so-setup | 2 ++ 2 files changed, 13 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index c9f0925ec..47d75f1c0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -44,6 +44,17 @@ logCmd() { } ### End Logging Section ### +airgap_repo() { + # Remove all the repo files + rm -rf /etc/yum.repos.d/* + echo "[airgap_repo]" > /etc/yum./repos.d/airgap_repo.repo + echo "baseurl=https://$MSRV/repo" >> /etc/yum.repos.d/airgap_repo.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/airgap_repo.repo + echo "sslverify=0" >> /etc/yum.repos.d/airgap_repo.repo + echo "name=Airgap Repo" >> /etc/yum.repos.d/airgap_repo.repo + echo "enabled=1" >> /etc/yum.repos.d/airgap_repo.repo +} + airgap_rules() { # Copy the rules for suricata if using Airgap mkdir -p /nsm/repo/rules diff --git a/setup/so-setup b/setup/so-setup index 0aa78aa10..d3ef48ba6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -648,6 +648,8 @@ set_redirect >> $setup_log 2>&1 if [[ ! $is_airgap ]]; then securityonion_repo >> $setup_log 2>&1 update_packages >> $setup_log 2>&1 + else + airgap_repo >> $setup_log 2>&1 fi if [[ $is_sensor || $is_helix || $is_import ]]; then From 83bf709290e14a845e68fc90234374356c22acdc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 15:12:53 -0400 Subject: [PATCH 083/188] use -r for salt boostrap in soup as well --- salt/common/tools/sbin/soup | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index cb2d19aed..d06002b7d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,11 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - if [ $is_airgap -eq 0 ]; then - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" - else - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - fi + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 89f72bb6edfc8437c3681e95b6c7b91e62c2da70 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 16:44:51 -0400 Subject: [PATCH 084/188] check if . in new version, append .1 if not --- salt/common/tools/sbin/soup | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d06002b7d..eda93fc1f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -649,6 +649,12 @@ fi echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded +SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'}) +if [[ "." =~ .*"$SALTVERSIONPOSTUPGRADE".* ]]; then + SALTVERSIONPOSTUPGRADE=$SALTVERSIONPOSTUPGRADE +else + SALTVERSIONPOSTUPGRADE="${SALTVERSIONPOSTUPGRADE}.1" +fi if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." From 5f6770925d0ede70182da4c510f244d3793e3b14 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Apr 2021 16:52:12 -0400 Subject: [PATCH 085/188] speculative commit --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 018bdfac7..e889c64d0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1008,7 +1008,7 @@ create_repo() { detect_cloud() { echo "Testing if setup is running on a cloud instance..." | tee -a "$setup_log" - if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null); then export is_cloud="true"; fi + if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null) || [ -f /var/log/waagent.log ]; then export is_cloud="true"; fi } detect_os() { From 3a1c478d9a3e9f0415eeae9f5e8fe5fbb1c94f11 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Apr 2021 16:56:34 -0400 Subject: [PATCH 086/188] compare the new var --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index eda93fc1f..9036d83bb 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -655,7 +655,7 @@ if [[ "." =~ .*"$SALTVERSIONPOSTUPGRADE".* ]]; then else SALTVERSIONPOSTUPGRADE="${SALTVERSIONPOSTUPGRADE}.1" fi -if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) != "$NEWSALTVERSION" ]]; then +if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." echo "Exiting." From a38015bd989e0101ecafa3eb9c4285bf94c35750 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 17:28:04 -0400 Subject: [PATCH 087/188] Add some manager logic --- salt/common/init.sls | 3 +++ setup/so-functions | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 5fe3d9081..0ada77e1a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -3,6 +3,7 @@ {% set role = grains.id.split('_') | last %} {% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} +{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -125,6 +126,7 @@ crwazrepo: file.absent: - name: /etc/yum.repos.d/wazuh.repo +{% if not ISAIRGAP %} crsecurityonionrepo: file.managed: {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} @@ -137,6 +139,7 @@ crsecurityonionrepo: - mode: 644 {% endif %} +{% endif %} # Install common packages {% if grains['os'] != 'CentOS' %} diff --git a/setup/so-functions b/setup/so-functions index 47d75f1c0..55cd9290d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -48,7 +48,11 @@ airgap_repo() { # Remove all the repo files rm -rf /etc/yum.repos.d/* echo "[airgap_repo]" > /etc/yum./repos.d/airgap_repo.repo - echo "baseurl=https://$MSRV/repo" >> /etc/yum.repos.d/airgap_repo.repo + if $is_manager; then + echo "baseurl=https://$HOSTNAME/repo" >> /etc/yum.repos.d/airgap_repo.repo + else + echo "baseurl=https://$MSRV/repo" >> /etc/yum.repos.d/airgap_repo.repo + fi echo "gpgcheck=1" >> /etc/yum.repos.d/airgap_repo.repo echo "sslverify=0" >> /etc/yum.repos.d/airgap_repo.repo echo "name=Airgap Repo" >> /etc/yum.repos.d/airgap_repo.repo From 168d0bcaf4c1bbee023868651d7cebd9a36cc0fd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 5 Apr 2021 18:30:07 -0400 Subject: [PATCH 088/188] Fix Spelling issue --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 55cd9290d..2e5e9ee7c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -47,7 +47,7 @@ logCmd() { airgap_repo() { # Remove all the repo files rm -rf /etc/yum.repos.d/* - echo "[airgap_repo]" > /etc/yum./repos.d/airgap_repo.repo + echo "[airgap_repo]" > /etc/yum.repos.d/airgap_repo.repo if $is_manager; then echo "baseurl=https://$HOSTNAME/repo" >> /etc/yum.repos.d/airgap_repo.repo else From 5525b9e97d17c472b58a23b42b6aecb8b2732d66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 08:30:57 -0400 Subject: [PATCH 089/188] point to new salt repo --- salt/common/yum_repos/securityonion.repo | 2 +- salt/common/yum_repos/securityonioncache.repo | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index e61829380..2fb35e579 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -42,7 +42,7 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..56ada1413 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -42,7 +42,7 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From 1ea0be00976a8f298b5e7c43b8c5cd54f270847c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 09:15:22 -0400 Subject: [PATCH 090/188] remove references to 3003.1 change to 3003 --- salt/common/tools/sbin/soup | 5 ----- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- salt/top.sls | 1 + setup/so-functions | 8 ++++---- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9036d83bb..0a68fae2d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -650,11 +650,6 @@ echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'}) -if [[ "." =~ .*"$SALTVERSIONPOSTUPGRADE".* ]]; then - SALTVERSIONPOSTUPGRADE=$SALTVERSIONPOSTUPGRADE -else - SALTVERSIONPOSTUPGRADE="${SALTVERSIONPOSTUPGRADE}.1" -fi if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index e97115ec1..8588af65c 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3003.1 \ No newline at end of file + version: 3003 \ No newline at end of file diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 3993c827b..9d888f106 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,5 +2,5 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3003.1 + version: 3003 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 68c392c25..6b522d03b 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -17,6 +17,7 @@ {% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {% import_yaml 'salt/minion.defaults.yaml' as saltversion %} {% set saltversion = saltversion.salt.minion.version %} +{% set INSTALLEDSALTVERSION = grains.saltversion %} base: diff --git a/setup/so-functions b/setup/so-functions index 8dd5d2f75..bc2908a41 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2033,7 +2033,7 @@ saltify() { fi set_progress_str 7 'Installing salt-master' if [[ ! $is_iso ]]; then - logCmd "yum -y install salt-master-3003.1" + logCmd "yum -y install salt-master-3003" fi systemctl enable salt-master >> "$setup_log" 2>&1 ;; @@ -2046,7 +2046,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' { if [[ ! $is_iso ]]; then - yum -y install salt-minion-3003.1\ + yum -y install salt-minion-3003\ python3\ python36-docker\ python36-dateutil\ @@ -2120,7 +2120,7 @@ saltify() { set_progress_str 6 'Installing various dependencies' retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1 set_progress_str 7 'Installing salt-master' - retry 50 10 "apt-get -y install salt-master=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-master=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 ;; *) @@ -2138,7 +2138,7 @@ saltify() { retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 set_progress_str 8 'Installing salt-minion & python modules' - retry 50 10 "apt-get -y install salt-minion=3003.1+ds-1 salt-common=3003.1+ds-1" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1 From 049001d572fd0ad5f5801c32a448782dd9d04ea0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 09:48:21 -0400 Subject: [PATCH 091/188] set repo url for salt upgrade for centos --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0a68fae2d..d8007012e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 01f95c846c6a9403f4b704646e6effaf8b3156c1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 11:41:06 -0400 Subject: [PATCH 092/188] remove trailing / --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d8007012e..f36cdac68 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 521dbbd90a1f7441c39deb4191013dbb0e569005 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 11:45:59 -0400 Subject: [PATCH 093/188] change repo path --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f36cdac68..e14c9a0aa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003/ -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 6da84c7c87fb2b1b2519b06100fa026d2bbbbbf5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 12:00:36 -0400 Subject: [PATCH 094/188] strip trailing / --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e14c9a0aa..74a1c26c0 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003/ -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003 -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 80509fbbc61987eecf7b0bccd17d7c59b454a1e4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 12:23:11 -0400 Subject: [PATCH 095/188] fix -R repo option --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 74a1c26c0..5108e73d3 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -509,7 +509,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R https://repo.securityonion.net/file/securityonion-repo/saltstack3003/py3/redhat/7/x86_64/archive/3003 -F -M -x python3 stable "$NEWSALTVERSION" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From b70d9c089275206daa36f35b0837a07f534a19a5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 6 Apr 2021 13:20:46 -0400 Subject: [PATCH 096/188] Add end summary and warning about SSH host key change --- setup/so-setup | 9 +- setup/so-whiptail | 203 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 48322f246..509ad419d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -572,14 +572,14 @@ fi if [[ $is_manager || $is_import ]]; then collect_so_allow; fi -whiptail_make_changes +# This block sets REDIRECTIT which is used by a function outside the below subshell +set_redirect >> $setup_log 2>&1 + +whiptail_end_settings # From here on changes will be made. echo "1" > /root/accept_changes -# This block sets REDIRECTIT which is used by a function outside the below subshell -set_redirect >> $setup_log 2>&1 - # Begin install { @@ -962,6 +962,7 @@ else } | whiptail_gauge_post_setup "Running post-installation steps..." whiptail_setup_complete + [[ $setup_type != 'iso' ]] && whitpail_ssh_warning echo "Post-installation steps have completed." >> $setup_log 2>&1 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index c71db9508..45b263f96 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -423,6 +423,193 @@ whiptail_enable_components() { done } +whiptail_end_settings() { + [ -n "$TESTING" ] && return + + # BASIC INFO (NETWORK, HOSTNAME, DESCRIPTION, ETC) + + read -r -d '' end_msg <<- EOM + The following options have been set, would you like to proceed? + + Node Type: $install_type + Hostname: $HOSTNAME + EOM + + [[ -n $NODE_DESCRIPTION ]] && __append_end_msg "Description: $NODE_DESCRIPTION" + + [[ $is_airgap ]] && __append_end_msg "Airgap: True" + + if [[ $is_minion ]]; then + __append_end_msg "Manager Hostname: $MSRV" + __append_end_msg "Manager IP: $MSRVIP" + fi + + + [[ $is_iso ]] && __append_end_msg "Network: $address_type" + + __append_end_msg "Management NIC: $MNIC" + __append_end_msg "Management IP: $MAINIP" + + if [[ $address_type == 'STATIC' ]]; then + __append_end_msg "Gateway: $MGATEWAY" + __append_end_msg "DNS: $MDNS" + __append_end_msg "DNS Domain: $MSEARCH" + fi + + if [[ $is_sensor ]]; then + __append_end_msg "Bond NIC(s):" + for nic in "${BNICS[@]}"; do + __append_end_msg " - $nic" + done + fi + + local homenet_arr + if [[ -n $HNMANAGER ]]; then + __append_end_msg "Home Network(s):" + IFS="," read -r -a homenet_arr <<< "$HNMANAGER" + for net in "${homenet_arr[@]}"; do + __append_end_msg " - $net" + done + elif [[ -n $HNSENSOR ]]; then + __append_end_msg "Home Network(s):" + IFS="," read -r -a homenet_arr <<< "$HNSENSOR" + for net in "${homenet_arr[@]}"; do + __append_end_msg " - $net" + done + fi + + [[ -n $REDIRECTIT ]] && __append_end_msg "Access URL: https://${REDIRECTIT}" + + [[ -n $ALLOW_CIDR ]] && __append_end_msg "Allowed IP or Subnet: $ALLOW_CIDR" + + [[ -n $WEBUSER ]] && __append_end_msg "Web User: $WEBUSER" + + [[ -n $FLEETNODEUSER ]] && __append_end_msg "Fleet User: $FLEETNODEUSER" + + if [[ $is_manager ]]; then + __append_end_msg "Enabled Optional Components:" + for component in "${COMPONENTS[@]}"; do + __append_end_msg " - $component" + done + fi + + # METADATA / IDS + + if [[ -n $ZEEKVERSION ]]; then + local md_tool_string=${ZEEKVERSION,;} + md_tool_string=${md_tool_string^} + + __append_end_msg "Metadata Tool: $md_tool_string" + fi + + [[ -n $RULESETUP ]] && __append_end_msg "IDS Ruleset: $RULESETUP" + [[ -n $OINKCODE ]] && __append_end_msg "Oinkcode: $OINKCODE" + + # PATCH SCHEDULE + + [[ -n $PATCHSCHEDULENAME ]] && __append_end_msg "Patch schedule: $PATCHSCHEDULENAME" + + if [[ ${#PATCHSCHEDULEDAYS[@]} -gt 0 ]]; then + __append_end_msg "Day(s):" + for day in "${PATCHSCHEDULEDAYS[@]}"; do + __append_end_msg " - $day" + done + fi + + if [[ ${#PATCHSCHEDULEHOURS[@]} -gt 0 ]]; then + __append_end_msg "Hours(s):" + for hour in "${PATCHSCHEDULEHOURS[@]}"; do + __append_end_msg " - $hour" + done + fi + + # MISC + + [[ $is_helix ]] && __append_end_msg "Helix API key: $HELIXAPIKEY" + [[ -n $DOCKERNET ]] && __append_end_msg "Docker network: $DOCKERNET" + if [[ -n $MANAGERUPDATES ]]; then + __append_end_msg "OS Package Updates: Manager" + else + __append_end_msg "OS Package Updates: Open" + fi + if [[ ${#ntp_servers[@]} -gt 0 ]]; then + __append_end_msg "NTP Servers:" + for server in "${ntp_servers[@]}"; do + __append_end_msg " - $server" + done + fi + + # ADVANCED OR REGULAR + + if [[ $NODESETUP == 'NODEADVANCED' ]]; then + __append_end_msg "Advanced Node Settings:" + __append_end_msg " Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE" + __append_end_msg " Logstash Heap Size: $NODE_LS_HEAP_SIZE" + __append_end_msg " Logstash Worker Count: $LSPIPELINEWORKERS" + __append_end_msg " Logstash Batch Size: $LSPIPELINEBATCH" + __append_end_msg " Logstash Input Threads: $LSINPUTTHREADS" + __append_end_msg " Curator Day Cutoff: $CURCLOSEDAYS days" + __append_end_msg " Elasticsearch Storage Space: ${log_size_limit}GB" + else + __append_end_msg "Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE" + __append_end_msg "Logstash Heap Size: $NODE_LS_HEAP_SIZE" + __append_end_msg "Logstash Worker Count: $LSPIPELINEWORKERS" + __append_end_msg "Logstash Batch Size: $LSPIPELINEBATCH" + __append_end_msg "Logstash Input Threads: $LSINPUTTHREADS" + __append_end_msg "Curator Close After: $CURCLOSEDAYS days" + __append_end_msg "Elasticsearch Storage Space: ${log_size_limit}GB" + fi + + + # ADVANCED + if [[ $MANAGERADV == 'ADVANCED' ]]; then + __append_end_msg "Advanced Manager Settings:" + __append_end_msg " ES Cluster Name: $ESCLUSTERNAME" + if [[ ${#BLOGS[@]} -gt 0 ]]; then + __append_end_msg " Zeek Logs Enabled:" + for log in "${BLOGS[@]}"; do + __append_end_msg " - $log" + done + fi + fi + + if [[ $NSMSETUP == 'ADVANCED' ]]; then + __append_end_msg "Advanced NSM Settings:" + if [[ ${#ZEEKPINS[@]} -gt 0 ]]; then + local zeek_pin_str + for core in "${ZEEKPINS[@]}"; do + zeek_pin_str="${zeek_pin_str}${core}," + done + zeek_pin_str=${zeek_pin_str%,} + __append_end_msg " Zeek Pinned Cores: ${zeek_pin_str}" + fi + if [[ ${#SURIPINS[@]} -gt 0 ]]; then + local suri_pin_str + for core in "${SURIPINS[@]}"; do + suri_pin_str="${suri_pin_str}${core}," + done + suri_pin_str=${suri_pin_str%,} + __append_end_msg " Suricata Pinned Cores: ${suri_pin_str}" + fi + else + [[ -n $BASICZEEK ]] && __append_end_msg " Zeek Processes: $BASICZEEK" + [[ -n $BASICSURI ]] && __append_end_msg " Suricata Processes: $BASICSURI" + fi + + whiptail --yesno "$end_msg" 24 75 --scrolltext + local exitstatus=$? + whiptail_check_exitstatus +} + +__append_end_msg() { + local newline=$1 + + read -r -d '' end_msg <<- EOM + $end_msg + $newline + EOM +} + whiptail_eval_adv() { [ -n "$TESTING" ] && return @@ -1491,6 +1678,22 @@ whiptail_so_allow() { whiptail_check_exitstatus $exitstatus } +whitpail_ssh_warning() { + [ -n "$TESTING" ] && return + + local msg + + read -r -d '' msg <<- EOM + NOTE: You will recceive a warning upon SSH reconnect that the host key has changed. + + This is expected due to hardening of the OpenSSH server config. + + The host key algorithm will now be ED25519, follow the instructions given by your SSH client to remove the old key fingerprint then retry the connection. + EOM + + whiptail --msgbox "$msg" 14 75 +} + whiptail_storage_requirements() { local mount=$1 local current_val=$2 From 6d6829ba340a86d9ef2150c34765c305dd1d1558 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 6 Apr 2021 13:21:07 -0400 Subject: [PATCH 097/188] Remove duplicate variable assignment --- setup/so-setup | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 509ad419d..5b1a7417c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -558,7 +558,6 @@ if [[ $is_node && ! $is_eval ]]; then LSPIPELINEWORKERS=$num_cpu_cores LSPIPELINEBATCH=125 LSINPUTTHREADS=1 - LSPIPELINEBATCH=125 fi fi From af6403f8746180fdd3c94e1223dff3760a4acadd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 15:45:05 -0400 Subject: [PATCH 098/188] soup salt and repos ohh my --- salt/common/init.sls | 6 ++++ salt/common/tools/sbin/soup | 28 +++++++++++++++++-- salt/common/yum_repos/securityonion.repo | 14 +++++----- salt/common/yum_repos/securityonioncache.repo | 4 +-- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 0ada77e1a..adf34a43a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -74,6 +74,12 @@ repair_yumdb: - onlyif: - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' +crsynckeys: + file.recurse: + - name: /etc/pki/rpm_gpg + - source: salt://common/keys/ + + crbase: file.absent: - name: /etc/yum.repos.d/CentOS-Base.repo diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index cb2d19aed..a24af62f9 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -409,6 +409,30 @@ up_2.3.2X_to_2.3.30() { sed -i "/^strelka:/a \\ repos: \n - https://github.com/Neo23x0/signature-base" /opt/so/saltstack/local/pillar/global.sls; fi check_log_size_limit + INSTALLEDVERSION=2.3.30 +} + +up_2.3.3X_to_2.3.50() { + if [[ $OS == 'centos' ]]; then + # Import GPG Keys + gpg_rpm_import + + if [[ ! $is_airgap ]]; then + + DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') + + for DELREPO in "${DELREPOS[@]}"; + rm /etc/yum.repos.d/$DELREPO + done + + # Copy the new repo file if not airgap + cp $UPDATE_DIR/salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + yum clean all + yum repolist + fi + fi + INSTALLEDVERSION=2.3.50 + } verify_upgradespace() { @@ -503,7 +527,7 @@ upgrade_salt() { echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." echo "" # If CentOS - if [ "$OS" == "centos" ]; then + if [[ $OS == 'centos' ]]; then echo "Removing yum versionlock for Salt." echo "" yum versionlock delete "salt-*" @@ -518,7 +542,7 @@ upgrade_salt() { echo "" yum versionlock add "salt-*" # Else do Ubuntu things - elif [ "$OS" == "ubuntu" ]; then + elif [[ $OS == 'ubuntu' ]]; then echo "Removing apt hold for Salt." echo "" apt-mark unhold "salt-common" diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index e61829380..0cd96bd91 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -31,25 +31,25 @@ name=Extra Packages for Enterprise Linux 7 - $basearch baseurl=https://repo.securityonion.net/file/securityonion-repo/epel/ enabled=1 gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/RPM-GPG-KEY-EPEL-7 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7 [docker-ce-stable] name=Docker CE Stable - $basearch baseurl=https://repo.securityonion.net/file/securityonion-repo/docker-ce-stable enabled=1 gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub +gpgkey=file:///etc/pki/rpm-gpg/docker.pub -[saltstack] +[saltstack3003] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +gpgkey=file:///etc/pki/rpm-gpg/SALTSTACK-GPG-KEY.pub [wazuh_repo] gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh_repo/ @@ -57,7 +57,7 @@ protect=1 [wazuh4_repo] gpgcheck=1 -gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-WAZUH enabled=1 name=Wazuh repository baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/ diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..bc0454ae7 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -40,9 +40,9 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub -[saltstack] +[saltstack3003] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From 92768ecd08e51d006064faf0beaf5081c3c0ad74 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 15:47:50 -0400 Subject: [PATCH 099/188] Add upgrade function --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index a24af62f9..6578432fa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -239,6 +239,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 + [[ "$INSTALLEDVERSION" == 2.3.30 ]] && up_2.3.3X_to_2.3.50 } postupgrade_changes() { From b50700114c4ee6a1ff0edbbd04419b6a447d91c1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 15:58:08 -0400 Subject: [PATCH 100/188] Add the do --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6578432fa..412f10e04 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -422,8 +422,8 @@ up_2.3.3X_to_2.3.50() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - for DELREPO in "${DELREPOS[@]}"; - rm /etc/yum.repos.d/$DELREPO + for DELREPO in "${DELREPOS[@]}"; do + rm /etc/yum.repos.d/"$DELREPO" done # Copy the new repo file if not airgap From 73e00dbe30dceeb5418932f49576643fbccdc2c9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Apr 2021 16:07:08 -0400 Subject: [PATCH 101/188] change salt upgrade in soup --- salt/common/tools/sbin/soup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c95f71499..a3fa70397 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -534,7 +534,11 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -R repo.securityonion.net/file/securityonion-repo/saltstack3003 -F -M -x python3 stable "$NEWSALTVERSION" + if [ $is_airgap -eq 0 ]; then + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + else + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + fi echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From 51bf988d31e00fa61f8228460b3e087e28d93088 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 16:21:19 -0400 Subject: [PATCH 102/188] Add .repo extension --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 412f10e04..b4cbc43ec 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -423,7 +423,7 @@ up_2.3.3X_to_2.3.50() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in "${DELREPOS[@]}"; do - rm /etc/yum.repos.d/"$DELREPO" + rm /etc/yum.repos.d/"$DELREPO".repo done # Copy the new repo file if not airgap From 6f7e6cee80bae73ef6c7f4d68124f6a2f0828eab Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 6 Apr 2021 16:43:42 -0400 Subject: [PATCH 103/188] Force it --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b4cbc43ec..46aa0c650 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -423,7 +423,7 @@ up_2.3.3X_to_2.3.50() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in "${DELREPOS[@]}"; do - rm /etc/yum.repos.d/"$DELREPO".repo + rm -f /etc/yum.repos.d/"$DELREPO".repo done # Copy the new repo file if not airgap From 099ac2ff19a86d2ffbf658a60b42266ba8e868aa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 09:06:22 -0400 Subject: [PATCH 104/188] Minor formatting changes to whiptail end screen --- setup/so-whiptail | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 45b263f96..e81c0be7c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -429,8 +429,6 @@ whiptail_end_settings() { # BASIC INFO (NETWORK, HOSTNAME, DESCRIPTION, ETC) read -r -d '' end_msg <<- EOM - The following options have been set, would you like to proceed? - Node Type: $install_type Hostname: $HOSTNAME EOM @@ -539,6 +537,11 @@ whiptail_end_settings() { done fi + if [[ $NSMSETUP != 'ADVANCED' ]]; then + [[ -n $BASICZEEK ]] && __append_end_msg "Zeek Processes: $BASICZEEK" + [[ -n $BASICSURI ]] && __append_end_msg "Suricata Processes: $BASICSURI" + fi + # ADVANCED OR REGULAR if [[ $NODESETUP == 'NODEADVANCED' ]]; then @@ -591,14 +594,12 @@ whiptail_end_settings() { suri_pin_str=${suri_pin_str%,} __append_end_msg " Suricata Pinned Cores: ${suri_pin_str}" fi - else - [[ -n $BASICZEEK ]] && __append_end_msg " Zeek Processes: $BASICZEEK" - [[ -n $BASICSURI ]] && __append_end_msg " Suricata Processes: $BASICSURI" fi - whiptail --yesno "$end_msg" 24 75 --scrolltext + whiptail --title "The following options have been set, would you like to proceed?" --yesno "$end_msg" 24 75 --scrolltext + local exitstatus=$? - whiptail_check_exitstatus + whiptail_check_exitstatus $exitstatus } __append_end_msg() { From 8f208728dde9fe1a4ef795da31c81357a60d41ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 09:10:16 -0400 Subject: [PATCH 105/188] change delete repos --- salt/common/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 5ce8d78c1..ffad5ad67 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -418,11 +418,11 @@ up_2.3.3X_to_2.3.50() { # Import GPG Keys gpg_rpm_import - if [[ ! $is_airgap ]]; then + if [ $is_airgap -eq 1 ]; then - DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') + DELREPOS=('CentOS-Base.repo' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - for DELREPO in "${DELREPOS[@]}"; do + for DELREPO in ${DELREPOS[@]}; do rm -f /etc/yum.repos.d/"$DELREPO".repo done From 5cd7d65b3f9c3d00115d4a181f86a4901abd1eff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 10:03:33 -0400 Subject: [PATCH 106/188] Fix Logic for Airgap distributed --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 214896572..9fef19875 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,7 +2285,7 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" ]]; then + if [[ "$OS" == "centos" && ! $is_airgap ]]; then mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ rm -f /etc/yum.repos.d/* From 88c565feae5934fe5a327201ec570188c802559c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 10:14:16 -0400 Subject: [PATCH 107/188] Fix proxy test logic --- setup/so-functions | 4 +++- setup/so-whiptail | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 8a751a4ad..e2e779775 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -563,7 +563,7 @@ collect_patch_schedule_name_import() { collect_proxy() { [[ -n $TESTING ]] && return - collect_proxy_details + collect_proxy_details || return while ! proxy_validate; do if whiptail_invalid_proxy; then collect_proxy_details no_ask @@ -608,6 +608,8 @@ collect_proxy_details() { so_proxy="$proxy_addr" fi export so_proxy + else + return 1 fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index e81c0be7c..fddf3b0fa 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -391,6 +391,7 @@ whiptail_dockernet_net() { whiptail_check_exitstatus $exitstatus } + whiptail_enable_components() { [ -n "$TESTING" ] && return From 5b3014496bdd493211bfd376f26dc91cddb1eece Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 10:35:59 -0400 Subject: [PATCH 108/188] Proxy fixes * Adjust proxy test timeout * Don't show proxy on error * Add echo statement so user knows what setup is doing --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e2e779775..0724f5851 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1872,12 +1872,13 @@ print_salt_state_apply() { } proxy_validate() { + echo "Testing proxy..." local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" - proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" 2>&1) + proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" --connect-timeout 5 2>&1) # set short connection timeout so user doesn't sit waiting for proxy test to timeout local ret=$? if [[ $ret != 0 ]]; then - error "Could not reach $test_url using proxy $so_proxy" + error "Could not reach $test_url using proxy provided" error "Received error: $proxy_test_err" if [[ -n $TESTING ]]; then error "Exiting setup" From 080ecba8e6e87700017f4ca365ff43db4bd62149 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 10:54:46 -0400 Subject: [PATCH 109/188] change delrepos --- salt/common/tools/sbin/soup | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ffad5ad67..0fc1017f6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -414,16 +414,18 @@ up_2.3.2X_to_2.3.30() { } up_2.3.3X_to_2.3.50() { - if [[ $OS == 'centos' ]]; then + if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import if [ $is_airgap -eq 1 ]; then - DELREPOS=('CentOS-Base.repo' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - + DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') + for DELREPO in ${DELREPOS[@]}; do - rm -f /etc/yum.repos.d/"$DELREPO".repo + if [[ -f "$DELREPO" ]]; then + rm -f /etc/yum.repos.d/"$DELREPO".repo + fi done # Copy the new repo file if not airgap @@ -433,7 +435,6 @@ up_2.3.3X_to_2.3.50() { fi fi INSTALLEDVERSION=2.3.50 - } verify_upgradespace() { From 54e039477623668ee5dc7a7858d13db191f6aca6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 10:57:09 -0400 Subject: [PATCH 110/188] change from saltstack3003 to just saltstack for repo --- salt/common/yum_repos/securityonion.repo | 4 ++-- salt/common/yum_repos/securityonioncache.repo | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index 0cd96bd91..9bb42552d 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -40,9 +40,9 @@ enabled=1 gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/docker.pub -[saltstack3003] +[saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ enabled=1 gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index bc0454ae7..6d5058337 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -40,9 +40,9 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub -[saltstack3003] +[saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From b7aa9ddaa3542c1c1d3724beac3444b7aebb0427 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 11:37:55 -0400 Subject: [PATCH 111/188] run preupgrade changes if 2.3.40 --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0fc1017f6..679c52a17 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -239,7 +239,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 - [[ "$INSTALLEDVERSION" == 2.3.30 ]] && up_2.3.3X_to_2.3.50 + [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50 } postupgrade_changes() { From ed0cd97de5af75600c3a4a2b2d95539de6378bca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 12:34:23 -0400 Subject: [PATCH 112/188] Fix Logic for Airgap distributed --- setup/so-functions | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 9fef19875..2e7a21797 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,15 +2285,17 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" && ! $is_airgap ]]; then - mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* - if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ - else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ - fi + if [[ "$OS" == "centos" ]]; then + if [[ ! $is_airgap ]]; then + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + fi + fi else echo "This is Ubuntu" fi From c8c1553247d6c07e441e5c62c6c8bced81f021dd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 12:36:50 -0400 Subject: [PATCH 113/188] Fix Logic for Airgap distributed --- setup/so-functions | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 2e7a21797..9aca9ca54 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,17 +2285,15 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" ]]; then - if [[ ! $is_airgap ]]; then - mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* - if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ - else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ - fi - fi + if [[ "$OS" == "centos" && $is_airgap ]]; then + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + fi else echo "This is Ubuntu" fi From 5578206bf119311a7b2ea7b99f8a0974b0054f81 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 12:41:01 -0400 Subject: [PATCH 114/188] need to make the repo changes before we try to upgrade sa;t --- salt/common/tools/sbin/soup | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 679c52a17..e9f337f65 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -230,6 +230,13 @@ masterunlock() { fi } +preupgrade_changes_2.3.50_repo() { + # We made repo changes in 2.3.50 and this prepares for that on upgrade + echo "Checking to see if 2.3.50 repo changes are needed." + + [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo +} + preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." @@ -413,7 +420,7 @@ up_2.3.2X_to_2.3.30() { INSTALLEDVERSION=2.3.30 } -up_2.3.3X_to_2.3.50() { +up_2.3.3X_to_2.3.50_repo() { if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import @@ -434,6 +441,9 @@ up_2.3.3X_to_2.3.50() { yum repolist fi fi +} + +up_2.3.3X_to_2.3.50() { INSTALLEDVERSION=2.3.50 } @@ -655,6 +665,7 @@ else update_registry update_docker_containers "soup" fi + echo "" echo "Stopping Salt Minion service." systemctl stop salt-minion @@ -665,6 +676,8 @@ echo "Stopping Salt Master service." systemctl stop salt-master echo "" +preupgrade_changes_2.3.50_repo + # Does salt need upgraded. If so update it. if [ "$UPGRADESALT" == "1" ]; then echo "Upgrading Salt" From 249fa06fc7f1c51475c7419dc86386d43d1e3639 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 13:03:27 -0400 Subject: [PATCH 115/188] echo when performing the repo actions for 2.3.50 --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e9f337f65..4584c207a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -421,12 +421,13 @@ up_2.3.2X_to_2.3.30() { } up_2.3.3X_to_2.3.50_repo() { + echo 'Performing 2.3.50 repo actions.' if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import if [ $is_airgap -eq 1 ]; then - + echo 'Deleting unneeded repo files.' DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in ${DELREPOS[@]}; do From ceb1ea61dcf25f3e9dfc021b11d0ff3e53240b40 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:15:49 -0400 Subject: [PATCH 116/188] Summary screen changes --- setup/so-whiptail | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index fddf3b0fa..eccf8c69c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -460,6 +460,7 @@ whiptail_end_settings() { for nic in "${BNICS[@]}"; do __append_end_msg " - $nic" done + __append_end_msg "MTU: $MTU" fi local homenet_arr @@ -506,20 +507,25 @@ whiptail_end_settings() { # PATCH SCHEDULE - [[ -n $PATCHSCHEDULENAME ]] && __append_end_msg "Patch schedule: $PATCHSCHEDULENAME" - - if [[ ${#PATCHSCHEDULEDAYS[@]} -gt 0 ]]; then - __append_end_msg "Day(s):" - for day in "${PATCHSCHEDULEDAYS[@]}"; do - __append_end_msg " - $day" - done - fi - - if [[ ${#PATCHSCHEDULEHOURS[@]} -gt 0 ]]; then - __append_end_msg "Hours(s):" - for hour in "${PATCHSCHEDULEHOURS[@]}"; do - __append_end_msg " - $hour" - done + if [[ -n $PATCHSCHEDULENAME ]]; then + __append_end_msg "Patch Schedule:" + if [[ $PATCHSCHEDULENAME != 'auto' && $PATCHSCHEDULENAME != 'manual' ]]; then + __append_end_msg " Type: $PATCHSCHEDULENAME" + else + __append_end_msg " Name: $PATCHSCHEDULENAME" + fi + if [[ ${#PATCHSCHEDULEDAYS[@]} -gt 0 ]]; then + __append_end_msg " Day(s):" + for day in "${PATCHSCHEDULEDAYS[@]}"; do + __append_end_msg " - $day" + done + fi + if [[ ${#PATCHSCHEDULEHOURS[@]} -gt 0 ]]; then + __append_end_msg " Hours(s):" + for hour in "${PATCHSCHEDULEHOURS[@]}"; do + __append_end_msg " - $hour" + done + fi fi # MISC From 377b14ccb1dab47eb8d6743cf626d047551f99fa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:20:55 -0400 Subject: [PATCH 117/188] ESCLUSTERNAME is empty for standalone, so check if it's set before listing --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index eccf8c69c..2522e65f9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -574,7 +574,7 @@ whiptail_end_settings() { # ADVANCED if [[ $MANAGERADV == 'ADVANCED' ]]; then __append_end_msg "Advanced Manager Settings:" - __append_end_msg " ES Cluster Name: $ESCLUSTERNAME" + [[ -n $ESCLUSTERNAME ]] && __append_end_msg " ES Cluster Name: $ESCLUSTERNAME" if [[ ${#BLOGS[@]} -gt 0 ]]; then __append_end_msg " Zeek Logs Enabled:" for log in "${BLOGS[@]}"; do From 425e5bc4c30dda35c27d730a1c465120bc05821b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 13:31:43 -0400 Subject: [PATCH 118/188] add some quotes --- salt/common/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 4584c207a..6824ac8a2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -421,18 +421,19 @@ up_2.3.2X_to_2.3.30() { } up_2.3.3X_to_2.3.50_repo() { - echo 'Performing 2.3.50 repo actions.' + echo "Performing 2.3.50 repo actions." if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import if [ $is_airgap -eq 1 ]; then - echo 'Deleting unneeded repo files.' + echo "Deleting unneeded repo files." DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') - for DELREPO in ${DELREPOS[@]}; do + for DELREPO in "${DELREPOS[@]}"; do if [[ -f "$DELREPO" ]]; then - rm -f /etc/yum.repos.d/"$DELREPO".repo + echo "Deleting $DELREPO.repo" + rm -f "/etc/yum.repos.d/$DELREPO.repo" fi done From f83ac5a2788089946ab5dc79a5bdb06e712bce04 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:38:47 -0400 Subject: [PATCH 119/188] Print install summary to file and setup log after user confirms --- setup/so-whiptail | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 2522e65f9..3e4a79a91 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -607,6 +607,9 @@ whiptail_end_settings() { local exitstatus=$? whiptail_check_exitstatus $exitstatus + + echo "$end_msg" > /root/install_summary + printf '%s\n' 'Install summary:' "$end_msg" >> "$setup_log" } __append_end_msg() { From ec076bba4ae58d11a92974b39d84cf44b80783e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:42:18 -0400 Subject: [PATCH 120/188] MTU is not always set by the user, so don't always show in summary --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 3e4a79a91..7c2665363 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -460,7 +460,7 @@ whiptail_end_settings() { for nic in "${BNICS[@]}"; do __append_end_msg " - $nic" done - __append_end_msg "MTU: $MTU" + [[ -n $MTU ]] && __append_end_msg "MTU: $MTU" fi local homenet_arr From c4f01192763e4f661eee745777a25f6802b1ed9a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Apr 2021 13:51:40 -0400 Subject: [PATCH 121/188] fix check if repo file exists --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6824ac8a2..6448c432f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -431,7 +431,7 @@ up_2.3.3X_to_2.3.50_repo() { DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh') for DELREPO in "${DELREPOS[@]}"; do - if [[ -f "$DELREPO" ]]; then + if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then echo "Deleting $DELREPO.repo" rm -f "/etc/yum.repos.d/$DELREPO.repo" fi From 3a4cf8aa269cc1dee056f27a4d54eb784fb5d1ae Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 13:54:01 -0400 Subject: [PATCH 122/188] Add proxy url/user to summary --- setup/so-whiptail | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 7c2665363..000aa2ba4 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -449,6 +449,14 @@ whiptail_end_settings() { __append_end_msg "Management NIC: $MNIC" __append_end_msg "Management IP: $MAINIP" + if [[ -n $so_proxy ]]; then + __append_end_msg "Proxy:" + __append_end_msg " Server URL: $proxy_addr" + [[ -n $proxy_user ]] && __append_end_msg " User: $proxy_user" + else + __append_end_msg "Proxy: N/A" + fi + if [[ $address_type == 'STATIC' ]]; then __append_end_msg "Gateway: $MGATEWAY" __append_end_msg "DNS: $MDNS" From 3c69c0c24cbb501e628dc40ab00cf50092589298 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 14:15:02 -0400 Subject: [PATCH 123/188] Correct patch schedule name logic in summary --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 000aa2ba4..33053a273 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -517,7 +517,7 @@ whiptail_end_settings() { if [[ -n $PATCHSCHEDULENAME ]]; then __append_end_msg "Patch Schedule:" - if [[ $PATCHSCHEDULENAME != 'auto' && $PATCHSCHEDULENAME != 'manual' ]]; then + if [[ $PATCHSCHEDULENAME == 'auto'|| $PATCHSCHEDULENAME == 'manual' ]]; then __append_end_msg " Type: $PATCHSCHEDULENAME" else __append_end_msg " Name: $PATCHSCHEDULENAME" From 9baa9767cafb722fb864bfd90e27e2622f6b497d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 16:12:51 -0400 Subject: [PATCH 124/188] Add raid bind --- salt/telegraf/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index 2814eb159..c4871a0b3 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -72,6 +72,7 @@ so-telegraf: - /opt/so/conf/telegraf/scripts:/scripts:ro - /opt/so/log/stenographer:/var/log/stenographer:ro - /opt/so/log/suricata:/var/log/suricata:ro + - /opt/so/log/raid:/var/log/raid:ro - watch: - file: tgrafconf - file: tgrafsyncscripts From 8ab4dd10d42ac97b522abb2345d45e00ba36f43c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 16:29:44 -0400 Subject: [PATCH 125/188] Add sostatus for telegraf --- salt/common/init.sls | 19 +++++++++++++++++++ salt/telegraf/init.sls | 1 + 2 files changed, 20 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index 0ada77e1a..9ee126ac1 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -297,6 +297,25 @@ commonlogrotateconf: - month: '*' - dayweek: '*' +# Create the status directory +sostatusdir: + file.directory: + - name: /opt/so/log/sostatus + - user: 0 + - group: 0 + - makedirs: True + +# Install sostatus check cron +/usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: + cron.present: + - user: root + - minute: '*/15' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + + {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %} # Lock permissions on the backup directory backupdir: diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index c4871a0b3..cea4d3f45 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -73,6 +73,7 @@ so-telegraf: - /opt/so/log/stenographer:/var/log/stenographer:ro - /opt/so/log/suricata:/var/log/suricata:ro - /opt/so/log/raid:/var/log/raid:ro + - /opt/so/log/sostatus:/var/log/sostatus:ro - watch: - file: tgrafconf - file: tgrafsyncscripts From 3caaf0682043074b1576d175f3ec1ba94f95b6bb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 7 Apr 2021 16:30:16 -0400 Subject: [PATCH 126/188] Add sostatus for telegraf --- salt/telegraf/scripts/sostatus.sh | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 salt/telegraf/scripts/sostatus.sh diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh new file mode 100644 index 000000000..23096d903 --- /dev/null +++ b/salt/telegraf/scripts/sostatus.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=sostatus +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf +SOSTATUSLOG=/var/log/sostatus/status.log +SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) + +if [ -f "$SOSTATUSLOG" ]; then + echo "sostatus status=$SOSTATUSSTATUS" +else + exit 0 +fi From a5f5888913e00ab59d3459701d01f37e6e357b1e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 7 Apr 2021 17:03:08 -0400 Subject: [PATCH 127/188] Summary order change --- setup/so-whiptail | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 33053a273..493ae7a68 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -449,6 +449,12 @@ whiptail_end_settings() { __append_end_msg "Management NIC: $MNIC" __append_end_msg "Management IP: $MAINIP" + if [[ $address_type == 'STATIC' ]]; then + __append_end_msg "Gateway: $MGATEWAY" + __append_end_msg "DNS: $MDNS" + __append_end_msg "DNS Domain: $MSEARCH" + fi + if [[ -n $so_proxy ]]; then __append_end_msg "Proxy:" __append_end_msg " Server URL: $proxy_addr" @@ -457,12 +463,6 @@ whiptail_end_settings() { __append_end_msg "Proxy: N/A" fi - if [[ $address_type == 'STATIC' ]]; then - __append_end_msg "Gateway: $MGATEWAY" - __append_end_msg "DNS: $MDNS" - __append_end_msg "DNS Domain: $MSEARCH" - fi - if [[ $is_sensor ]]; then __append_end_msg "Bond NIC(s):" for nic in "${BNICS[@]}"; do From d4a3bc455071b90b36b9233fb7392b8fdbd4cfa9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 08:43:20 -0400 Subject: [PATCH 128/188] Fix so repo for salt --- salt/common/yum_repos/securityonion.repo | 7 +++++++ salt/common/yum_repos/securityonioncache.repo | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index e61829380..0f39d5a3f 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -47,6 +47,13 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +[saltstack3003] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + [wazuh_repo] gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..def6f8a40 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -47,6 +47,13 @@ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub +[saltstack3003] +name=SaltStack repo for RHEL/CentOS $releasever PY3 +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ +enabled=1 +gpgcheck=1 +gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub + [wazuh_repo] gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH From dce476b604e30923ff79e566eea3593bbbdc6a30 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 8 Apr 2021 09:54:41 -0400 Subject: [PATCH 129/188] change back to saltstack3003 repo --- salt/common/yum_repos/securityonion.repo | 2 +- salt/common/yum_repos/securityonioncache.repo | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/yum_repos/securityonion.repo b/salt/common/yum_repos/securityonion.repo index 9bb42552d..5fd9b5fc4 100644 --- a/salt/common/yum_repos/securityonion.repo +++ b/salt/common/yum_repos/securityonion.repo @@ -42,7 +42,7 @@ gpgkey=file:///etc/pki/rpm-gpg/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/common/yum_repos/securityonioncache.repo index 6d5058337..56ada1413 100644 --- a/salt/common/yum_repos/securityonioncache.repo +++ b/salt/common/yum_repos/securityonioncache.repo @@ -42,7 +42,7 @@ gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub [saltstack] name=SaltStack repo for RHEL/CentOS $releasever PY3 -baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack/ +baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/ enabled=1 gpgcheck=1 gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub From 725320ebc8880629f3c2550a21e55b1ba3785e52 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:02:11 -0400 Subject: [PATCH 130/188] Fix Repo Logic --- salt/common/init.sls | 2 +- setup/so-functions | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 9ee126ac1..93f76c3b3 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -309,7 +309,7 @@ sostatusdir: /usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: cron.present: - user: root - - minute: '*/15' + - minute: '*/5' - hour: '*' - daymonth: '*' - month: '*' diff --git a/setup/so-functions b/setup/so-functions index 9aca9ca54..54f8d2abf 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2285,14 +2285,18 @@ secrets_pillar(){ securityonion_repo() { # Remove all the current repos - if [[ "$OS" == "centos" && $is_airgap ]]; then - mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* - if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + if [[ "$OS" == "centos" ]]; then + if [[ "$INTERWEBS" == "AIRGAP" ]]; then + echo "This is airgap I don't need to add this repo" else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + mkdir -p /root/oldrepos + mv /etc/yum.repos.d/* /root/oldrepos/ + rm -f /etc/yum.repos.d/* + if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then + cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + else + cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + fi fi else echo "This is Ubuntu" From ce9f781d81ea51ad265b630e5f79f1959db161ab Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:24:04 -0400 Subject: [PATCH 131/188] Fix Repo Logic --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 54f8d2abf..1633d0901 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2291,7 +2291,7 @@ securityonion_repo() { else mkdir -p /root/oldrepos mv /etc/yum.repos.d/* /root/oldrepos/ - rm -f /etc/yum.repos.d/* + rm -rf /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else From 951369c2d698cbd9a7a738c839c099ffae55d430 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:25:36 -0400 Subject: [PATCH 132/188] Fix Repo Logic --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 1633d0901..0771567b2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2290,7 +2290,7 @@ securityonion_repo() { echo "This is airgap I don't need to add this repo" else mkdir -p /root/oldrepos - mv /etc/yum.repos.d/* /root/oldrepos/ + mv -v /etc/yum.repos.d/* /root/oldrepos/ rm -rf /etc/yum.repos.d/* if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ From fdaf251ba02111b21244524018d7dd6a1b13c6a4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:36:52 -0400 Subject: [PATCH 133/188] Fix Repo Logic --- setup/so-functions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 0771567b2..b8cd2cae2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2291,7 +2291,9 @@ securityonion_repo() { else mkdir -p /root/oldrepos mv -v /etc/yum.repos.d/* /root/oldrepos/ - rm -rf /etc/yum.repos.d/* + ls -la /etc/yum.repos.d/ + rm -rf /etc/yum.repos.d + mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ else From 4c5f373ffa2c3e5f3532d32e42c1d50c1e88e6a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:37:44 -0400 Subject: [PATCH 134/188] Fix Repo Logic --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index b8cd2cae2..f2face572 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2293,6 +2293,7 @@ securityonion_repo() { mv -v /etc/yum.repos.d/* /root/oldrepos/ ls -la /etc/yum.repos.d/ rm -rf /etc/yum.repos.d + yum clean all mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ From 09b14e6a863ede8568d0b0622f3a18dd41af9eb1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 10:38:50 -0400 Subject: [PATCH 135/188] Fix Repo Logic --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index f2face572..9cbad1cfb 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2294,6 +2294,7 @@ securityonion_repo() { ls -la /etc/yum.repos.d/ rm -rf /etc/yum.repos.d yum clean all + yum repolist all mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ From f7f95b6c541c6db3a5264f1094da7b604b3ff617 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 8 Apr 2021 11:22:54 -0400 Subject: [PATCH 136/188] Add model to sensoroni agent config --- salt/sensoroni/files/sensoroni.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index df2990404..378d42373 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,9 +1,6 @@ {%- set URLBASE = salt['pillar.get']('global:url_base') %} -{%- if salt['pillar.get']('sensoroni:node_description') %} -{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %} -{%- else %} -{%- set DESCRIPTION = salt['grains.get']('sosmodel', '') %} -{%- endif %} +{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description', '') %} +{%- set MODEL = salt['grains.get']('sosmodel', '') %} {%- set ADDRESS = salt['pillar.get']('sensoroni:node_address') %} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} {%- set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) %} @@ -21,6 +18,7 @@ "role": "{{ grains.role }}", "description": "{{ DESCRIPTION }}", "address": "{{ ADDRESS }}", + "model": "{{ MODEL }}", "pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }}, "serverUrl": "https://{{ URLBASE }}/sensoroniagents", "verifyCert": false, From b53815d04af3731f59fdf466653f040715d10f7f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 8 Apr 2021 11:42:41 -0400 Subject: [PATCH 137/188] Fix Telegraf sostatus --- salt/telegraf/etc/telegraf.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index f6bcbdaf5..1b172485b 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -663,6 +663,15 @@ # # Read metrics from one or more commands that can output to stdout +[[inputs.exec]] + commands = [ + "/scripts/sostatus.sh" + ] + data_format = "influx" + timeout = "15s" + interval = "180s" + + # ## Commands array {% if grains['role'] in ['so-manager', 'so-managersearch'] %} [[inputs.exec]] From 6650ad5cdd545e8378a945edf7abfd94b92f85d5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 8 Apr 2021 14:04:30 -0400 Subject: [PATCH 138/188] make the -r for all --- salt/common/tools/sbin/soup | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6448c432f..70d453334 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -547,11 +547,7 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - if [ $is_airgap -eq 0 ]; then - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" - else - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - fi + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" From c9feda116853bf91d5160c8205dc4602268c6afe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 9 Apr 2021 08:48:29 -0400 Subject: [PATCH 139/188] Do not upgrade salt on ISO installs --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index b7acf7cb1..ba815e57f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2718,7 +2718,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then yum repolist >> /dev/null - yum -y update >> "$setup_log" + yum -y update --exclude=salt* >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 From b2fcd438c26d699da9a556fc7558c207caf6927a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 09:39:33 -0400 Subject: [PATCH 140/188] Initial support for checking state of manager during setup --- salt/common/init.sls | 5 +++++ setup/so-functions | 19 +++++++++++++++++++ setup/so-setup | 4 ++++ setup/so-whiptail | 17 +++++++++++++++-- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 93f76c3b3..3ce6286be 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -304,6 +304,11 @@ sostatusdir: - user: 0 - group: 0 - makedirs: True + +sostatus_log: + file.managed: + - name: /opt/so/log/sostatus/status.log + - mode: 644 # Install sostatus check cron /usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: diff --git a/setup/so-functions b/setup/so-functions index b7acf7cb1..604eb56b7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -162,6 +162,25 @@ check_hive_init() { docker rm so-thehive } +check_manager_state() { + echo "Checking state of manager services. This may take a moment..." + retry 2 15 "__check_so_status" && retry 2 15 "__check_salt_master" && return 0 || return 1 +} + +__check_so_status() { + local so_status_output + so_status_output=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /opt/so/log/sostatus/status.log) + [[ -z $so_status_output ]] && so_status_output=1 + return $so_status_output +} + +__check_salt_master() { + local salt_master_status + salt_master_status=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" systemctl is-active --quiet salt-master) + [[ -z $salt_master_status ]] && salt_master_status=1 + return $salt_master_status +} + check_network_manager_conf() { local gmdconf="/usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf" local nmconf="/etc/NetworkManager/NetworkManager.conf" diff --git a/setup/so-setup b/setup/so-setup index 5b1a7417c..79c9b9784 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -574,6 +574,10 @@ if [[ $is_manager || $is_import ]]; then collect_so_allow; fi # This block sets REDIRECTIT which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 +if [[ $is_minion ]] && ! check_manager_state; then + whiptail_manager_error || exit 1 +fi + whiptail_end_settings # From here on changes will be made. diff --git a/setup/so-whiptail b/setup/so-whiptail index 493ae7a68..2b1199fb5 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -79,7 +79,7 @@ whiptail_bond_nics_mtu() { whiptail_cancel() { - whiptail --title "Security Onion Setup" --msgbox "Cancelling Setup. No changes have been made." 8 75 + whiptail --title "Security Onion Setup" --msgbox "Cancelling Setup." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -88,7 +88,7 @@ whiptail_cancel() { } >> $setup_log 2>&1 fi - title "User cancelled setup, no changes made." + title "User cancelled setup." exit } @@ -1140,6 +1140,19 @@ whiptail_manager_adv_service_zeeklogs() { } +whiptail_manager_error() { + local msg + read -r -d '' <<- EOM + Setup could not determine if the manager $MSRV is in a good state. + + Continuing without verifying all services on the manager are running may result in a failure. + + Would you like to continue anyway? + EOM + + whiptail --title "Security Onion Setup" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 +} + whiptail_manager_updates() { [ -n "$TESTING" ] && return From 9b5276f1ab401a65f99de480304d8bfd1174ec8a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 09:59:54 -0400 Subject: [PATCH 141/188] Remove bad `||` statement --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 79c9b9784..84fcf5c34 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -575,7 +575,7 @@ if [[ $is_manager || $is_import ]]; then collect_so_allow; fi set_redirect >> $setup_log 2>&1 if [[ $is_minion ]] && ! check_manager_state; then - whiptail_manager_error || exit 1 + whiptail_manager_error fi whiptail_end_settings From fc9df2bbaec84bc8d6a1d780074de4cd588f0019 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:00:50 -0400 Subject: [PATCH 142/188] Update airgap question to ask during minion installs too --- setup/so-setup | 2 +- setup/so-whiptail | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 84fcf5c34..f53909bdc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -264,7 +264,7 @@ elif [ "$install_type" = 'ANALYST' ]; then fi # Check if this is an airgap install -if [[ ( $is_manager || $is_import ) && $is_iso ]]; then +if [[ $is_iso || $is_minion ]]; then whiptail_airgap if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true diff --git a/setup/so-whiptail b/setup/so-whiptail index 2b1199fb5..368b6b0cd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -19,10 +19,13 @@ whiptail_airgap() { [ -n "$TESTING" ] && return + local node_str='node' + [[ $is_manager || $is_import ]] && node_str='manager' + INTERWEBS=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose your install conditions:" 20 75 4 \ - "STANDARD" "This manager has internet accesss" ON \ - "AIRGAP" "This manager does not have internet access" OFF 3>&1 1>&2 2>&3 ) + "STANDARD" "This $node_str has internet accesss" ON \ + "AIRGAP" "This $node_str does not have internet access" OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 764307bfa055a667f00b8fd5fae5b173c5407103 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:09:28 -0400 Subject: [PATCH 143/188] Reformat airgap whiptail prompt --- setup/so-whiptail | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 368b6b0cd..1fbfa34dd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -23,12 +23,14 @@ whiptail_airgap() { [[ $is_manager || $is_import ]] && node_str='manager' INTERWEBS=$(whiptail --title "Security Onion Setup" --radiolist \ - "Choose your install conditions:" 20 75 4 \ - "STANDARD" "This $node_str has internet accesss" ON \ - "AIRGAP" "This $node_str does not have internet access" OFF 3>&1 1>&2 2>&3 ) + "How should this $node_str be installed?" 10 60 2 \ + "Standard " "This $node_str has internet accesss." ON \ + "Airgap " "This $node_str does not have internet access." OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus + + INTERWEBS=$(echo "${INTERWEBS^^}" | tr -d ' ') } whiptail_avoid_default_hostname() { From 026ce769666f11a4e439dc70149654863a0cd1e9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:11:00 -0400 Subject: [PATCH 144/188] Change airgap prompt to menu --- setup/so-whiptail | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 1fbfa34dd..50b50e353 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -22,10 +22,10 @@ whiptail_airgap() { local node_str='node' [[ $is_manager || $is_import ]] && node_str='manager' - INTERWEBS=$(whiptail --title "Security Onion Setup" --radiolist \ + INTERWEBS=$(whiptail --title "Security Onion Setup" --menu \ "How should this $node_str be installed?" 10 60 2 \ - "Standard " "This $node_str has internet accesss." ON \ - "Airgap " "This $node_str does not have internet access." OFF 3>&1 1>&2 2>&3 ) + "Standard " "This $node_str has internet accesss" \ + "Airgap " "This $node_str does not have internet access" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 4db20a00fff56b2be682cb4199c975acfb83dfc7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Apr 2021 10:16:19 -0400 Subject: [PATCH 145/188] Add quotes around description, since it can contain spaces --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index f53909bdc..bac2fcdfd 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -339,7 +339,7 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ "MSRVIP=$MSRVIP" \ - "NODE_DESCRIPTION=$NODE_DESCRIPTION" > "$install_opt_file" + "NODE_DESCRIPTION=\"$NODE_DESCRIPTION\"" > "$install_opt_file" [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" From 5cb73ced36ef40f6683be14db1d5ec4391fd8328 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 9 Apr 2021 14:58:15 -0400 Subject: [PATCH 146/188] Add Influx module to SOC config --- salt/soc/files/soc/soc.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 6e2850aca..1dcd46c63 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -53,6 +53,13 @@ "cacheMs": {{ ES_FIELDCAPS_CACHE }}, "verifyCert": false }, + "influxdb": { + "hostUrl": "https://{{ MANAGERIP }}:8086", + "token": "", + "org": "", + "bucket": "telegraf", + "verifyCert": false + }, "sostatus": { "refreshIntervalMs": 30000, "offlineThresholdMs": 900000 From 8facbcf18c16b432fdb2beef12cbfc023c8abead Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 9 Apr 2021 20:40:44 -0400 Subject: [PATCH 147/188] Do not set influxdb hostUrl if import node since import nodes don't run influxdb --- salt/soc/files/soc/soc.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 1dcd46c63..6f1c3a6da 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -54,7 +54,11 @@ "verifyCert": false }, "influxdb": { +{%- if grains['role'] in ['so-import'] %} + "hostUrl": "", +{%- else %} "hostUrl": "https://{{ MANAGERIP }}:8086", +{%- endif %} "token": "", "org": "", "bucket": "telegraf", From 5d98c896a3791434491afdedc1de6a55fbf87a40 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 12:53:17 -0400 Subject: [PATCH 148/188] /opt/so/log needs 755 permissions for soremote to read sostatus log --- salt/common/init.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index 3ce6286be..ee7cad5e6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -51,6 +51,11 @@ sosaltstackperms: - gid: 939 - dir_mode: 770 +so_log_perms: + file.directory: + - name: /opt/so/log + - dir_mode: 755 + # Create a state directory statedir: file.directory: From 73a1bdd885f7943ee52538e025282dcc736101fa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 12:59:45 -0400 Subject: [PATCH 149/188] Send stdout to log, and actually populate error message --- setup/so-functions | 2 +- setup/so-whiptail | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 604eb56b7..5f07bb3b0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -164,7 +164,7 @@ check_hive_init() { check_manager_state() { echo "Checking state of manager services. This may take a moment..." - retry 2 15 "__check_so_status" && retry 2 15 "__check_salt_master" && return 0 || return 1 + retry 2 15 "__check_so_status" >> $setup_log 2>&1 && retry 2 15 "__check_salt_master" >> $setup_log 2>&1 && return 0 || return 1 } __check_so_status() { diff --git a/setup/so-whiptail b/setup/so-whiptail index 50b50e353..6ce2c214c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1147,7 +1147,7 @@ whiptail_manager_adv_service_zeeklogs() { whiptail_manager_error() { local msg - read -r -d '' <<- EOM + read -r -d '' msg <<- EOM Setup could not determine if the manager $MSRV is in a good state. Continuing without verifying all services on the manager are running may result in a failure. From 9240d376f3588d4029b14ff93400c2031bde9cf7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 12 Apr 2021 14:31:41 -0400 Subject: [PATCH 150/188] combine client repo management into 1 state --- salt/airgap/init.sls | 71 ---------------- salt/common/init.sls | 83 ------------------- .../client/files/centos/airgap}/yum.conf | 0 .../client/files/centos}/keys/GPG-KEY-WAZUH | 0 .../files/centos}/keys/RPM-GPG-KEY-EPEL-7 | 0 .../files/centos}/keys/SALTSTACK-GPG-KEY.pub | 0 .../client/files/centos}/keys/docker.pub | 0 .../files/centos}/keys/securityonion.pub | 0 .../client/files/centos}/securityonion.repo | 0 .../files/centos}/securityonioncache.repo | 0 .../client/files/centos}/yum.conf.jinja | 0 salt/repo/client/init.sls | 77 +++++++++++++++++ salt/repo/client/map.jinja | 25 ++++++ salt/top.sls | 11 +-- salt/yum/init.sls | 17 ---- 15 files changed, 104 insertions(+), 180 deletions(-) delete mode 100644 salt/airgap/init.sls rename salt/{airgap/files => repo/client/files/centos/airgap}/yum.conf (100%) rename salt/{common => repo/client/files/centos}/keys/GPG-KEY-WAZUH (100%) rename salt/{common => repo/client/files/centos}/keys/RPM-GPG-KEY-EPEL-7 (100%) rename salt/{common => repo/client/files/centos}/keys/SALTSTACK-GPG-KEY.pub (100%) rename salt/{common => repo/client/files/centos}/keys/docker.pub (100%) rename salt/{common => repo/client/files/centos}/keys/securityonion.pub (100%) rename salt/{common/yum_repos => repo/client/files/centos}/securityonion.repo (100%) rename salt/{common/yum_repos => repo/client/files/centos}/securityonioncache.repo (100%) rename salt/{yum/etc => repo/client/files/centos}/yum.conf.jinja (100%) create mode 100644 salt/repo/client/init.sls create mode 100644 salt/repo/client/map.jinja delete mode 100644 salt/yum/init.sls diff --git a/salt/airgap/init.sls b/salt/airgap/init.sls deleted file mode 100644 index 4ff401099..000000000 --- a/salt/airgap/init.sls +++ /dev/null @@ -1,71 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - -{% set MANAGER = salt['grains.get']('master') %} -airgapyum: - file.managed: - - name: /etc/yum/yum.conf - - source: salt://airgap/files/yum.conf - -airgap_repo: - pkgrepo.managed: - - humanname: Airgap Repo - - baseurl: https://{{ MANAGER }}/repo - - gpgcheck: 1 - - sslverify: 0 - -agbase: - file.absent: - - name: /etc/yum.repos.d/CentOS-Base.repo - -agcr: - file.absent: - - name: /etc/yum.repos.d/CentOS-CR.repo - -agdebug: - file.absent: - - name: /etc/yum.repos.d/CentOS-Debuginfo.repo - -agfasttrack: - file.absent: - - name: /etc/yum.repos.d/CentOS-fasttrack.repo - -agmedia: - file.absent: - - name: /etc/yum.repos.d/CentOS-Media.repo - -agsources: - file.absent: - - name: /etc/yum.repos.d/CentOS-Sources.repo - -agvault: - file.absent: - - name: /etc/yum.repos.d/CentOS-Vault.repo - -agkernel: - file.absent: - - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo - -agepel: - file.absent: - - name: /etc/yum.repos.d/epel.repo - -agtesting: - file.absent: - - name: /etc/yum.repos.d/epel-testing.repo - -agssrepo: - file.absent: - - name: /etc/yum.repos.d/saltstack.repo - -agwazrepo: - file.absent: - - name: /etc/yum.repos.d/wazuh.repo - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} \ No newline at end of file diff --git a/salt/common/init.sls b/salt/common/init.sls index b630891f5..0492b6535 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -2,8 +2,6 @@ {% if sls in allowed_states %} {% set role = grains.id.split('_') | last %} -{% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -66,87 +64,6 @@ salttmp: - group: 939 - makedirs: True -# Remove default Repos -{% if grains['os'] == 'CentOS' %} -repair_yumdb: - cmd.run: - - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all' - - onlyif: - - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' - -crsynckeys: - file.recurse: - - name: /etc/pki/rpm_gpg - - source: salt://common/keys/ - - -crbase: - file.absent: - - name: /etc/yum.repos.d/CentOS-Base.repo - -crcr: - file.absent: - - name: /etc/yum.repos.d/CentOS-CR.repo - -crdebug: - file.absent: - - name: /etc/yum.repos.d/CentOS-Debuginfo.repo - -crdockerce: - file.absent: - - name: /etc/yum.repos.d/docker-ce.repo - -crfasttrack: - file.absent: - - name: /etc/yum.repos.d/CentOS-fasttrack.repo - -crmedia: - file.absent: - - name: /etc/yum.repos.d/CentOS-Media.repo - -crsources: - file.absent: - - name: /etc/yum.repos.d/CentOS-Sources.repo - -crvault: - file.absent: - - name: /etc/yum.repos.d/CentOS-Vault.repo - -crkernel: - file.absent: - - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo - -crepel: - file.absent: - - name: /etc/yum.repos.d/epel.repo - -crtesting: - file.absent: - - name: /etc/yum.repos.d/epel-testing.repo - -crssrepo: - file.absent: - - name: /etc/yum.repos.d/saltstack.repo - -crwazrepo: - file.absent: - - name: /etc/yum.repos.d/wazuh.repo - -{% if not ISAIRGAP %} -crsecurityonionrepo: - file.managed: - {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} - - name: /etc/yum.repos.d/securityonion.repo - - source: salt://common/yum_repos/securityonion.repo - {% else %} - - name: /etc/yum.repos.d/securityonioncache.repo - - source: salt://common/yum_repos/securityonioncache.repo - {% endif %} - - mode: 644 - -{% endif %} -{% endif %} - # Install common packages {% if grains['os'] != 'CentOS' %} commonpkgs: diff --git a/salt/airgap/files/yum.conf b/salt/repo/client/files/centos/airgap/yum.conf similarity index 100% rename from salt/airgap/files/yum.conf rename to salt/repo/client/files/centos/airgap/yum.conf diff --git a/salt/common/keys/GPG-KEY-WAZUH b/salt/repo/client/files/centos/keys/GPG-KEY-WAZUH similarity index 100% rename from salt/common/keys/GPG-KEY-WAZUH rename to salt/repo/client/files/centos/keys/GPG-KEY-WAZUH diff --git a/salt/common/keys/RPM-GPG-KEY-EPEL-7 b/salt/repo/client/files/centos/keys/RPM-GPG-KEY-EPEL-7 similarity index 100% rename from salt/common/keys/RPM-GPG-KEY-EPEL-7 rename to salt/repo/client/files/centos/keys/RPM-GPG-KEY-EPEL-7 diff --git a/salt/common/keys/SALTSTACK-GPG-KEY.pub b/salt/repo/client/files/centos/keys/SALTSTACK-GPG-KEY.pub similarity index 100% rename from salt/common/keys/SALTSTACK-GPG-KEY.pub rename to salt/repo/client/files/centos/keys/SALTSTACK-GPG-KEY.pub diff --git a/salt/common/keys/docker.pub b/salt/repo/client/files/centos/keys/docker.pub similarity index 100% rename from salt/common/keys/docker.pub rename to salt/repo/client/files/centos/keys/docker.pub diff --git a/salt/common/keys/securityonion.pub b/salt/repo/client/files/centos/keys/securityonion.pub similarity index 100% rename from salt/common/keys/securityonion.pub rename to salt/repo/client/files/centos/keys/securityonion.pub diff --git a/salt/common/yum_repos/securityonion.repo b/salt/repo/client/files/centos/securityonion.repo similarity index 100% rename from salt/common/yum_repos/securityonion.repo rename to salt/repo/client/files/centos/securityonion.repo diff --git a/salt/common/yum_repos/securityonioncache.repo b/salt/repo/client/files/centos/securityonioncache.repo similarity index 100% rename from salt/common/yum_repos/securityonioncache.repo rename to salt/repo/client/files/centos/securityonioncache.repo diff --git a/salt/yum/etc/yum.conf.jinja b/salt/repo/client/files/centos/yum.conf.jinja similarity index 100% rename from salt/yum/etc/yum.conf.jinja rename to salt/repo/client/files/centos/yum.conf.jinja diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls new file mode 100644 index 000000000..60353426f --- /dev/null +++ b/salt/repo/client/init.sls @@ -0,0 +1,77 @@ +{% from 'repo/client/map.jinja' import ABSENTFILES with context %} +{% from 'repo/client/map.jinja' import REPOPATH with context %} +{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} +{% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %} +{% set role = grains.id.split('_') | last %} + +# from airgap state +{% if ISAIRGAP and grains.os == 'CentOS' %} +{% set MANAGER = salt['grains.get']('master') %} +airgapyum: + file.managed: + - name: /etc/yum/yum.conf + - source: salt://repo/client/files/centos/airgap/yum.conf + +airgap_repo: + pkgrepo.managed: + - humanname: Airgap Repo + - baseurl: https://{{ MANAGER }}/repo + - gpgcheck: 1 + - sslverify: 0 +{% endif %} + +# from airgap and common +{% if ABSENTFILES|length > 0%} + {% for file in ABSENTFILES %} +{{ file }}: + file.absent: + - name: {{ REPOPATH }}{{ file }} + - onchanges_in: cleanyum + {% endfor %} +{% endif %} + +# from common state +# Remove default Repos +{% if grains['os'] == 'CentOS' %} +repair_yumdb: + cmd.run: + - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all' + - onlyif: + - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"' + +crsynckeys: + file.recurse: + - name: /etc/pki/rpm_gpg + - source: salt://repo/client/files/centos/keys/ + +{% if not ISAIRGAP %} +crsecurityonionrepo: + file.managed: + {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %} + - name: /etc/yum.repos.d/securityonion.repo + - source: salt://repo/client/files/centos/securityonion.repo + {% else %} + - name: /etc/yum.repos.d/securityonioncache.repo + - source: salt://repo/client/files/centos/securityonioncache.repo + {% endif %} + - mode: 644 + +yumconf: + file.managed: + - name: /etc/yum.conf + - source: salt:/repo/client/files/centos/yum.conf.jinja + - mode: 644 + - template: jinja +{% endif %} + +cleanyum: + module.run: + - pkg.clean_metadata + - onchanges: + - file: airgapyum + - pkgrepo: airgap_repo + - file: crsecurityonionrepo + - file: yumconf + +{% endif %} + diff --git a/salt/repo/client/map.jinja b/salt/repo/client/map.jinja new file mode 100644 index 000000000..ccfa1eae2 --- /dev/null +++ b/salt/repo/client/map.jinja @@ -0,0 +1,25 @@ +{% if grains.os == 'CentOS' %} + + {% set REPOPATH = '/etc/yum.repos.d/' %} + {% set ABSENTFILES = [ + 'CentOS-Base.repo', + 'CentOS-CR.repo', + 'CentOS-Debuginfo.repo', + 'CentOS-fasttrack.repo', + 'CentOS-Media.repo', + 'CentOS-Sources.repo', + 'CentOS-Vault.repo', + 'CentOS-x86_64-kernel.repo', + 'epel.repo', + 'epel-testing.repo', + 'saltstack.repo', + 'wazuh.repo' + ] + %} + +{% elif grains.os == 'Ubuntu' %} + + {% set REPOPATH = '/etc/apt/sources.list.d/' %} + {% set ABSENTFILES = [] %} + +{% endif %} \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 6b522d03b..8a12aaa26 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -14,7 +14,6 @@ {% set CURATOR = salt['pillar.get']('curator:enabled', True) %} {% set REDIS = salt['pillar.get']('redis:enabled', True) %} {% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {% import_yaml 'salt/minion.defaults.yaml' as saltversion %} {% set saltversion = saltversion.salt.minion.version %} {% set INSTALLEDSALTVERSION = grains.saltversion %} @@ -24,18 +23,12 @@ base: 'not G@saltversion:{{saltversion}}': - match: compound - salt.minion-state-apply-test - {% if ISAIRGAP is sameas true %} - - airgap - {% endif %} + - repo.client - salt.minion 'G@os:CentOS and G@saltversion:{{saltversion}}': - match: compound - {% if ISAIRGAP is sameas true %} - - airgap - {% else %} - - yum - {% endif %} + - repo.client - yum.packages '* and G@saltversion:{{saltversion}}': diff --git a/salt/yum/init.sls b/salt/yum/init.sls deleted file mode 100644 index 339a6f2a7..000000000 --- a/salt/yum/init.sls +++ /dev/null @@ -1,17 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - -yumconf: - file.managed: - - name: /etc/yum.conf - - source: salt://yum/etc/yum.conf.jinja - - mode: 644 - - template: jinja - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} \ No newline at end of file From cf4de255eccddbef4a44c79116de9a8bd04dc567 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Apr 2021 15:18:18 -0400 Subject: [PATCH 151/188] Fix Wazuh WEL Shipping --- salt/elasticsearch/files/ingest/win.eventlogs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/win.eventlogs b/salt/elasticsearch/files/ingest/win.eventlogs index 2644be7a2..ec5cf911c 100644 --- a/salt/elasticsearch/files/ingest/win.eventlogs +++ b/salt/elasticsearch/files/ingest/win.eventlogs @@ -4,8 +4,8 @@ { "set": { "if": "ctx.winlog?.channel != null", "field": "event.module", "value": "windows_eventlog", "override": false, "ignore_failure": true } }, { "set": { "if": "ctx.winlog?.channel != null", "field": "event.dataset", "value": "{{winlog.channel}}", "override": true } }, { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, - { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, - { "set": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "value": "{{winlog.systemTime}}", "override": true } }, + { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, + { "date": { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"] } }, { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, { "set": { "field": "event.category", "value": "host", "override": true } }, { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_failure": true, "ignore_missing": true } }, From cc344d921ac481d17e5d2e6f4179215a46e1ca9e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 16:13:32 -0400 Subject: [PATCH 152/188] Skip whiptail during testing, echo error message to setup log --- setup/so-setup | 1 + setup/so-whiptail | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index bac2fcdfd..ad210048a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -575,6 +575,7 @@ if [[ $is_manager || $is_import ]]; then collect_so_allow; fi set_redirect >> $setup_log 2>&1 if [[ $is_minion ]] && ! check_manager_state; then + echo "Manager was not in a good state" >> "$setup_log" 2>&1 whiptail_manager_error fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 6ce2c214c..6f1cc6171 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1146,6 +1146,9 @@ whiptail_manager_adv_service_zeeklogs() { } whiptail_manager_error() { + + [ -n "$TESTING" ] && return + local msg read -r -d '' msg <<- EOM Setup could not determine if the manager $MSRV is in a good state. @@ -1159,9 +1162,6 @@ whiptail_manager_error() { } whiptail_manager_updates() { - - [ -n "$TESTING" ] && return - local update_string update_string=$(whiptail --title "Security Onion Setup" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ From 25637b74dbece9886e23489555ddb80b395466f5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 16:14:47 -0400 Subject: [PATCH 153/188] Add back removed testing skip --- setup/so-whiptail | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 6f1cc6171..6127a174a 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1162,6 +1162,9 @@ whiptail_manager_error() { } whiptail_manager_updates() { + + [ -n "$TESTING" ] && return + local update_string update_string=$(whiptail --title "Security Onion Setup" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ From 9d09e7bec3cbfa0d192e4df4a1594483ffbb5155 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 12 Apr 2021 16:25:17 -0400 Subject: [PATCH 154/188] Fix sostatus log cron job --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index ee7cad5e6..f7a4d6731 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -316,7 +316,7 @@ sostatus_log: - mode: 644 # Install sostatus check cron -/usr/sbin/so-status -q && echo $? > /opt/so/log/sostatus/status.log 2>&1: +'/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1': cron.present: - user: root - minute: '*/5' From eb7bf58f30c9bc688b799d4a3c2e1acc608bc469 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 12 Apr 2021 16:33:32 -0400 Subject: [PATCH 155/188] fix issues with repo.client state --- salt/repo/client/init.sls | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/repo/client/init.sls b/salt/repo/client/init.sls index 60353426f..5567caac2 100644 --- a/salt/repo/client/init.sls +++ b/salt/repo/client/init.sls @@ -26,7 +26,8 @@ airgap_repo: {{ file }}: file.absent: - name: {{ REPOPATH }}{{ file }} - - onchanges_in: cleanyum + - onchanges_in: + - module: cleanyum {% endfor %} {% endif %} @@ -59,19 +60,22 @@ crsecurityonionrepo: yumconf: file.managed: - name: /etc/yum.conf - - source: salt:/repo/client/files/centos/yum.conf.jinja + - source: salt://repo/client/files/centos/yum.conf.jinja - mode: 644 - template: jinja {% endif %} cleanyum: module.run: - - pkg.clean_metadata + - pkg.clean_metadata: [] - onchanges: +{% if ISAIRGAP %} - file: airgapyum - pkgrepo: airgap_repo +{% else %} - file: crsecurityonionrepo - file: yumconf +{% endif %} {% endif %} From 325264dafd858413ff6f977e4fba8d9b20f34d0a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 12 Apr 2021 17:44:50 -0400 Subject: [PATCH 156/188] point to new repo location --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 70d453334..2a1ddab1c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -438,7 +438,7 @@ up_2.3.3X_to_2.3.50_repo() { done # Copy the new repo file if not airgap - cp $UPDATE_DIR/salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ yum clean all yum repolist fi From eb94c011e22ec056501b5aee66da88a39054a5f5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Apr 2021 11:15:15 -0400 Subject: [PATCH 157/188] update location of yum keys and repo files for setup --- salt/common/tools/sbin/so-common | 4 ++-- setup/so-functions | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index a2c28587d..97e61e6e2 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -165,9 +165,9 @@ get_random_value() { gpg_rpm_import() { if [[ "$OS" == "centos" ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then - local RPMKEYSLOC="../salt/common/keys" + local RPMKEYSLOC="../salt/repo/client/files/centos/keys" else - local RPMKEYSLOC="$UPDATEDIR/salt/common/keys" + local RPMKEYSLOC="$UPDATEDIR/salt/repo/client/files/centos/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') diff --git a/setup/so-functions b/setup/so-functions index d3a695fa1..ae65c7547 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2300,9 +2300,9 @@ securityonion_repo() { yum repolist all mkdir -p /etc/yum.repos.d if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then - cp -f ../salt/common/yum_repos/securityonioncache.repo /etc/yum.repos.d/ + cp -f ../salt/repo/client/files/centos/securityonioncache.repo /etc/yum.repos.d/ else - cp -f ../salt/common/yum_repos/securityonion.repo /etc/yum.repos.d/ + cp -f ../salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ fi fi else From 7cbeed985a836f6a77a0904beff5c79f81652f6e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 13 Apr 2021 12:55:40 -0400 Subject: [PATCH 158/188] Differentiate between event & ingest timestamp --- salt/elasticsearch/files/ingest/osquery.query_result | 1 + salt/elasticsearch/files/ingest/win.eventlogs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result index 9bb381946..a58df1315 100644 --- a/salt/elasticsearch/files/ingest/osquery.query_result +++ b/salt/elasticsearch/files/ingest/osquery.query_result @@ -9,6 +9,7 @@ { "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } }, { "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } }, { "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.datetime", "target_field": "winlog.systemTime", "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } }, { "set": { "field": "event.module", "value": "osquery", "override": false } }, diff --git a/salt/elasticsearch/files/ingest/win.eventlogs b/salt/elasticsearch/files/ingest/win.eventlogs index ec5cf911c..a6ef87256 100644 --- a/salt/elasticsearch/files/ingest/win.eventlogs +++ b/salt/elasticsearch/files/ingest/win.eventlogs @@ -4,8 +4,8 @@ { "set": { "if": "ctx.winlog?.channel != null", "field": "event.module", "value": "windows_eventlog", "override": false, "ignore_failure": true } }, { "set": { "if": "ctx.winlog?.channel != null", "field": "event.dataset", "value": "{{winlog.channel}}", "override": true } }, { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, - { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, - { "date": { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"] } }, + { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, + { "date": { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'","yyyy-MM-dd'T'HH:mm:ss.SSSSSSS'Z'"] } }, { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, { "set": { "field": "event.category", "value": "host", "override": true } }, { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_failure": true, "ignore_missing": true } }, From 989c2b23b11540a310c86f1d11acafe197871ea4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 15:34:03 -0400 Subject: [PATCH 159/188] Fix Airgap Automation --- setup/automation/distributed-airgap-search | 1 + setup/automation/distributed-airgap-sensor | 1 + 2 files changed, 2 insertions(+) diff --git a/setup/automation/distributed-airgap-search b/setup/automation/distributed-airgap-search index 7a0888fee..1acee9b1a 100644 --- a/setup/automation/distributed-airgap-search +++ b/setup/automation/distributed-airgap-search @@ -35,6 +35,7 @@ ADMINPASS2=onionuser HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit HOSTNAME=distributed-search +INTERWEBS=AIRGAP install_type=SEARCHNODE # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= diff --git a/setup/automation/distributed-airgap-sensor b/setup/automation/distributed-airgap-sensor index 91b9c24a9..c8186bf8a 100644 --- a/setup/automation/distributed-airgap-sensor +++ b/setup/automation/distributed-airgap-sensor @@ -35,6 +35,7 @@ ZEEKVERSION=ZEEK HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit HOSTNAME=distributed-sensor +INTERWEBS=AIRGAP install_type=SENSOR # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= From 621e5c1cf819d797b77168dff12174a0fa8cc7e2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 19:18:10 -0400 Subject: [PATCH 160/188] Enable Filebeat Stats --- salt/filebeat/etc/filebeat.yml | 5 +++-- salt/filebeat/init.sls | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index c680d61c1..f6d3c5334 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -493,12 +493,13 @@ setup.template.enabled: false # append ?pretty to the URL. # Defines if the HTTP endpoint is enabled. -#http.enabled: false +http.enabled: true # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. -#http.host: localhost +http.host: localhost # Port on which the HTTP endpoint will bind. Default is 5066. +http.port: 5066 queue.mem.events: {{ FBMEMEVENTS }} queue.mem.flush.min_events: {{ FBMEMFLUSHMINEVENTS }} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 339d307ee..4d7f81819 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -74,6 +74,7 @@ so-filebeat: - port_bindings: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp + - 0.0.0.0:5066/tcp - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml From 7153f58a03cba46e5f45b334449e4944663e9350 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:17:26 -0400 Subject: [PATCH 161/188] Add Firewall for Beats port --- salt/filebeat/init.sls | 2 +- salt/firewall/portgroups.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 4d7f81819..64cdc47fc 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -74,7 +74,7 @@ so-filebeat: - port_bindings: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp - - 0.0.0.0:5066/tcp + - 0.0.0.0:5066:5066/tcp - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml diff --git a/salt/firewall/portgroups.yaml b/salt/firewall/portgroups.yaml index 55a09c6bf..1386267f5 100644 --- a/salt/firewall/portgroups.yaml +++ b/salt/firewall/portgroups.yaml @@ -18,6 +18,9 @@ firewall: beats_5644: tcp: - 5644 + beats_5066: + tcp: + - 5066 cortex: tcp: - 9001 From db7dcd76cdd969ce00ec3a6077a20407b0fdf8d2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:21:32 -0400 Subject: [PATCH 162/188] Add hostname to the listener --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index f6d3c5334..ecb16ac06 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -496,7 +496,7 @@ setup.template.enabled: false http.enabled: true # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. -http.host: localhost +http.host: {{ HOSTNAME }} # Port on which the HTTP endpoint will bind. Default is 5066. http.port: 5066 From aa66b6226fbe121c655a138346de6bce04119ef0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:22:51 -0400 Subject: [PATCH 163/188] Add hostname to the listener --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index ecb16ac06..0f7c9c778 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -496,7 +496,7 @@ setup.template.enabled: false http.enabled: true # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. -http.host: {{ HOSTNAME }} +http.host: 0.0.0.0 # Port on which the HTTP endpoint will bind. Default is 5066. http.port: 5066 From 904d34977f5508c5debf3b2ab6e306714a799e2c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:48:53 -0400 Subject: [PATCH 164/188] Add telegraf scripts to track eps and failures for beats --- salt/telegraf/etc/telegraf.conf | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 1b172485b..03113a3f4 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -679,7 +679,8 @@ "/scripts/redis.sh", "/scripts/influxdbsize.sh", "/scripts/eps.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" ## Timeout for each command to complete. @@ -695,7 +696,8 @@ "/scripts/zeekcaptureloss.sh", {% endif %} "/scripts/oldpcap.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -711,7 +713,8 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/eps.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -729,7 +732,8 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/eps.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -746,7 +750,8 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/influxdbsize.sh", - "/scripts/raid.sh" + "/scripts/raid.sh", + "/beatseps.sh" ] data_format = "influx" timeout = "15s" From 6fc7ed1a25b437bde17774bbdf309ab4bc3068b4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 13 Apr 2021 20:51:27 -0400 Subject: [PATCH 165/188] Add telegraf scripts to track eps and failures for beats --- salt/telegraf/scripts/beatseps.sh | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 salt/telegraf/scripts/beatseps.sh diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh new file mode 100644 index 000000000..faba0fabc --- /dev/null +++ b/salt/telegraf/scripts/beatseps.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=beatseps +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +PREVCOUNTFILE='/tmp/beatseps.txt' +EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" +FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" + +if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi + + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + +fi + +exit 0 From a50b3e84755369484d2b5e18c6e672e3898c5d9e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 10:22:06 -0400 Subject: [PATCH 166/188] add delay to salt-minion service starting - https://github.com/Security-Onion-Solutions/securityonion/issues/3543 --- salt/salt/minion.defaults.yaml | 3 ++- salt/salt/minion.sls | 17 +++++++++++- salt/salt/service/salt-minion.service.jinja | 30 +++++++++++++++++++++ salt/systemd/reload.sls | 3 +++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 salt/salt/service/salt-minion.service.jinja create mode 100644 salt/systemd/reload.sls diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 9d888f106..560493bed 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -3,4 +3,5 @@ salt: minion: version: 3003 - check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default \ No newline at end of file + check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default + service_start_delay: 30 # in seconds. \ No newline at end of file diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 004fddebe..e4e339b42 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,9 +2,11 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} +{% from 'salt/minion.defaults.yaml' import salt.minion.service_start_delay as service_start_delay %} include: - salt + - systemd.reload install_salt_minion: cmd.run: @@ -32,8 +34,21 @@ set_log_levels: - listen_in: - service: salt_minion_service +salt_minion_service_unit_file: + file.managed: + - name: /etc/systemd/system/multi-user.target.wants/salt-minion.service + - source: salt://salt/service/salt-minion.servic.jinja + - template: jinja + - defaults: + - service_start_delay: {{ service_start_delay }} + - onchanges_in: + - module: systemd_reload + - listen_in: + - service: salt_minion_service + salt_minion_service: service.running: - name: salt-minion - enable: True - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" \ No newline at end of file + - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" + diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja new file mode 100644 index 000000000..aea68b994 --- /dev/null +++ b/salt/salt/service/salt-minion.service.jinja @@ -0,0 +1,30 @@ +[Unit] +Description=The Salt Minion +Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html +After=network.target salt-master.service + +[Service] +KillMode=process +Type=notify +NotifyAccess=all +LimitNOFILE=8192 +ExecStart=/usr/bin/salt-minion +ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} + +[Install] +WantedBy=multi-user.target + + + + + + + + + + + + + + + diff --git a/salt/systemd/reload.sls b/salt/systemd/reload.sls new file mode 100644 index 000000000..ff2185539 --- /dev/null +++ b/salt/systemd/reload.sls @@ -0,0 +1,3 @@ +systemd_reload: + module.run: + - service.systemctl_reload: [] \ No newline at end of file From 9bfdae9cd5c9bb8f97694a57db80374a6d32e3f5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:06:06 -0400 Subject: [PATCH 167/188] fix import --- salt/salt/minion.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index e4e339b42..a2b8a17e3 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,7 +2,8 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} -{% from 'salt/minion.defaults.yaml' import salt.minion.service_start_delay as service_start_delay %} +{% from 'salt/minion.defaults.yaml' import salt as SALTMINION %} +{% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} include: - salt From 47aa66876d27c4d99cf12da999ea934542918e60 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:07:16 -0400 Subject: [PATCH 168/188] fix import --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index a2b8a17e3..2c023cbdd 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,7 +2,7 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} -{% from 'salt/minion.defaults.yaml' import salt as SALTMINION %} +{% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} {% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} include: From 48c531bc2c6de762e2859ff7e65bd8a977aa3266 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:09:13 -0400 Subject: [PATCH 169/188] fix file defaults def --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 2c023cbdd..9d16846d9 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -41,7 +41,7 @@ salt_minion_service_unit_file: - source: salt://salt/service/salt-minion.servic.jinja - template: jinja - defaults: - - service_start_delay: {{ service_start_delay }} + service_start_delay: {{ service_start_delay }} - onchanges_in: - module: systemd_reload - listen_in: From d003d4941b56e100a8eae657d206df68ee3a5a17 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:10:19 -0400 Subject: [PATCH 170/188] fix bad typing --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 9d16846d9..44f866880 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -38,7 +38,7 @@ set_log_levels: salt_minion_service_unit_file: file.managed: - name: /etc/systemd/system/multi-user.target.wants/salt-minion.service - - source: salt://salt/service/salt-minion.servic.jinja + - source: salt://salt/service/salt-minion.service.jinja - template: jinja - defaults: service_start_delay: {{ service_start_delay }} From f60da54ff030c2db9da65f87f4cbeb4a55c67c75 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 11:11:13 -0400 Subject: [PATCH 171/188] remove extra lines at end --- salt/salt/service/salt-minion.service.jinja | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja index aea68b994..c7bae0bc2 100644 --- a/salt/salt/service/salt-minion.service.jinja +++ b/salt/salt/service/salt-minion.service.jinja @@ -12,19 +12,4 @@ ExecStart=/usr/bin/salt-minion ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} [Install] -WantedBy=multi-user.target - - - - - - - - - - - - - - - +WantedBy=multi-user.target \ No newline at end of file From 71d7ca8958bdd68e01a98949049910a8577ca734 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Apr 2021 15:48:33 -0400 Subject: [PATCH 172/188] only manage service file if the right salt version is installed --- salt/salt/minion.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 44f866880..1c7f1a5e8 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -46,6 +46,7 @@ salt_minion_service_unit_file: - module: systemd_reload - listen_in: - service: salt_minion_service + - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" salt_minion_service: service.running: From f61bf73f972eafce84fd71fb92c73323c547b153 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 15 Apr 2021 11:59:34 +0000 Subject: [PATCH 173/188] Remove ESUSER/ESPASS for now to prevent issues with attempting Elastic Auth when connecting to ES. --- salt/soctopus/files/SOCtopus.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index c9941c3e1..b6ee45e74 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -8,8 +8,8 @@ [es] es_url = https://{{MANAGER}}:9200 es_ip = {{MANAGER}} -es_user = YOURESUSER -es_pass = YOURESPASS +es_user = +es_pass = es_index_pattern = so-* es_verifycert = no From 2f198ed9fb8b5646e469432fbdf6dac47d17abea Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 09:42:00 -0400 Subject: [PATCH 174/188] change how salt is held and unheld from updates --- salt/salt/map.jinja | 14 +++----------- salt/salt/minion.sls | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 6b5273b84..67742812b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -1,5 +1,6 @@ {% import_yaml 'salt/minion.defaults.yaml' as saltminion %} {% set SALTVERSION = saltminion.salt.minion.version %} +{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} @@ -7,20 +8,11 @@ {% set SPLITCHAR = '-' %} {% endif %} -{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} - -{% if grains.os|lower == 'ubuntu' %} - {% set COMMON = 'salt-common' %} -{% elif grains.os|lower in ['centos', 'redhat'] %} - {% set COMMON = 'salt' %} -{% endif %} - {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} - {% set UPGRADECOMMAND = 'yum clean all ; yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' ; yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common ; apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' ; apt-mark hold salt-common && apt-mark hold salt-minion' %} + {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 1c7f1a5e8..6488124f6 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -9,6 +9,12 @@ include: - salt - systemd.reload +{% if "{{INSTALLEDSALTVERSION}}" != "{{SALTVERSION}}" %} +unhold_salt_packages: + module.run: + - pkg.unhold: + - 'salt-*' + install_salt_minion: cmd.run: - name: | @@ -16,15 +22,13 @@ install_salt_minion: exec 1>&- # close stdout exec 2>&- # close stderr nohup /bin/sh -c '{{ UPGRADECOMMAND }}' & - - onlyif: test "{{INSTALLEDSALTVERSION}}" != "{{SALTVERSION}}" +{% endif %} -salt_minion_package: - pkg.installed: - - pkgs: - - {{ COMMON }} - - salt-minion - - hold: True - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" +{% if "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" %} +hold_salt_packages: + module.run: + - pkg.hold: + - 'salt-*' set_log_levels: file.append: @@ -46,11 +50,9 @@ salt_minion_service_unit_file: - module: systemd_reload - listen_in: - service: salt_minion_service - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" salt_minion_service: service.running: - name: salt-minion - enable: True - - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" - +{% endif %} From 22edbcc1112cbec4f93a525443c8de292448336e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 11:29:01 -0400 Subject: [PATCH 175/188] can use SPLITCHAR before defined --- salt/salt/map.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 67742812b..3ba7194f5 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -1,6 +1,5 @@ {% import_yaml 'salt/minion.defaults.yaml' as saltminion %} {% set SALTVERSION = saltminion.salt.minion.version %} -{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} @@ -8,6 +7,8 @@ {% set SPLITCHAR = '-' %} {% endif %} +{% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} + {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %} From 9d01387a04725611b3c7b96cafbd9bebe48070c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 11:57:25 -0400 Subject: [PATCH 176/188] remove references to the common salt package --- salt/salt/master.sls | 12 ++++-------- salt/salt/minion.sls | 1 - 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 3c23bbb36..d0a655051 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -1,17 +1,13 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% from 'salt/map.jinja' import COMMON with context %} - include: - salt.minion -salt_master_package: - pkg.installed: - - pkgs: - - {{ COMMON }} - - salt-master - - hold: True +hold_salt_master_package: + module.run: + - pkg.hold: + - 'salt-master' salt_master_service: service.running: diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 6488124f6..e656ae8a6 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -1,4 +1,3 @@ -{% from 'salt/map.jinja' import COMMON with context %} {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} From 9d676efada5b07a38d2140d387b6d1b4072b1a2b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Apr 2021 12:45:34 -0400 Subject: [PATCH 177/188] move salt_minion_service state outside jinja if --- salt/salt/minion.sls | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index e656ae8a6..1b0f7d901 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -49,9 +49,11 @@ salt_minion_service_unit_file: - module: systemd_reload - listen_in: - service: salt_minion_service +{% endif %} +# this has to be outside the if statement above since there are _in calls to this state salt_minion_service: service.running: - name: salt-minion - enable: True -{% endif %} + - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" \ No newline at end of file From 24b263c81227052bceaa468ac2175e7683ddeb1d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 16 Apr 2021 11:37:18 -0400 Subject: [PATCH 178/188] only hold/unhold packages if not already unheld/held --- salt/salt/map.jinja | 2 ++ salt/salt/minion.sls | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 3ba7194f5..5c1689e6c 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -3,8 +3,10 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} + {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?') %} {% else %} {% set SPLITCHAR = '-' %} + {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?') %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 1b0f7d901..5145da34b 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -1,6 +1,7 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} +{% from 'salt/map.jinja' import SALTNOTHELD %} {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} {% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} @@ -8,11 +9,14 @@ include: - salt - systemd.reload -{% if "{{INSTALLEDSALTVERSION}}" != "{{SALTVERSION}}" %} +{% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} + +{% if SALTNOTHELD == 0 %} unhold_salt_packages: module.run: - pkg.unhold: - - 'salt-*' + - name: 'salt-*' +{% endif %} install_salt_minion: cmd.run: @@ -23,11 +27,14 @@ install_salt_minion: nohup /bin/sh -c '{{ UPGRADECOMMAND }}' & {% endif %} -{% if "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" %} +{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} + +{% if SALTNOTHELD == 1 %} hold_salt_packages: module.run: - pkg.hold: - - 'salt-*' + - name: 'salt-*' +{% endif %} set_log_levels: file.append: From 1b15f018742d7f28bcce7b76582ec189fd9c3aea Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 16 Apr 2021 13:09:01 -0400 Subject: [PATCH 179/188] fix salt.master state --- salt/salt/master.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/salt/master.sls b/salt/salt/master.sls index d0a655051..8b2b6c7d0 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -1,13 +1,16 @@ +{% from 'salt/map.jinja' import SALTNOTHELD %} {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} include: - salt.minion +{% if SALTNOTHELD == 1 %} hold_salt_master_package: module.run: - pkg.hold: - - 'salt-master' + - name: salt-master +{% endif %} salt_master_service: service.running: From 58febe795574fd200bcb5e48c77659c75c8fd4e5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 16 Apr 2021 16:04:07 -0400 Subject: [PATCH 180/188] [fix] so-docker-prune breaks when multiple "so-" images share a version --- salt/common/tools/sbin/so-docker-prune | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 5a56f506d..f6c043ef3 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -60,15 +60,19 @@ def main(quiet): no_prunable = True for t_list in grouped_tag_lists: try: - # Keep the 2 most current images + # Group tags by version, in case multiple images exist with the same version string t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True) - if len(t_list) <= 2: + grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ] + + # Keep the 2 most current version groups + if len(grouped_t_list) <= 2: continue else: no_prunable = False - for tag in t_list[2:]: - if not quiet: print(f'Removing image {tag}') - client.images.remove(tag) + for group in grouped_t_list[2:]: + for tag in group: + if not quiet: print(f'Removing image {tag}') + client.images.remove(tag) except InvalidVersion as e: print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr) exit(1) From ef0669aabb8c0fb685ce2742a0ec1545891fbfb8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sat, 17 Apr 2021 18:24:33 -0400 Subject: [PATCH 181/188] Fix beat script location --- salt/telegraf/etc/telegraf.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 03113a3f4..0798fc920 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -680,7 +680,7 @@ "/scripts/influxdbsize.sh", "/scripts/eps.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" ## Timeout for each command to complete. @@ -697,7 +697,7 @@ {% endif %} "/scripts/oldpcap.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -714,7 +714,7 @@ "/scripts/oldpcap.sh", "/scripts/eps.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -733,7 +733,7 @@ "/scripts/oldpcap.sh", "/scripts/eps.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" @@ -751,7 +751,7 @@ "/scripts/oldpcap.sh", "/scripts/influxdbsize.sh", "/scripts/raid.sh", - "/beatseps.sh" + "/scripts/beatseps.sh" ] data_format = "influx" timeout = "15s" From 9e57fd2df0b88f18bc02629fe185dc47c0716169 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 19 Apr 2021 09:00:30 -0400 Subject: [PATCH 182/188] cant pipe to grep without , python_shell=True --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5c1689e6c..5d6d980be 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -3,10 +3,10 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} - {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?') %} + {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} {% else %} {% set SPLITCHAR = '-' %} - {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?') %} + {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} From 31f193c397c26cc2cdf7d088f1c30795048d602a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 12:36:46 -0400 Subject: [PATCH 183/188] Change EPS for Telegraf --- salt/telegraf/etc/telegraf.conf | 41 ++++++++++++++++-- salt/telegraf/scripts/beatseps.sh | 50 ++++++++++------------ salt/telegraf/scripts/checkfiles.sh | 18 ++++---- salt/telegraf/scripts/eps.sh | 50 ++++++++++------------ salt/telegraf/scripts/helixeps.sh | 48 ++++++++++----------- salt/telegraf/scripts/influxdbsize.sh | 18 ++++---- salt/telegraf/scripts/oldpcap.sh | 24 +++++------ salt/telegraf/scripts/raid.sh | 20 ++++----- salt/telegraf/scripts/redis.sh | 20 ++++----- salt/telegraf/scripts/sostatus.sh | 24 +++++------ salt/telegraf/scripts/stenoloss.sh | 50 +++++++++++----------- salt/telegraf/scripts/suriloss.sh | 49 ++++++++++------------ salt/telegraf/scripts/zeekcaptureloss.sh | 53 +++++++++++------------- salt/telegraf/scripts/zeekloss.sh | 53 +++++++++++------------- 14 files changed, 255 insertions(+), 263 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0798fc920..dac9bf60e 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -17,6 +17,7 @@ {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} +{%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} # Global tags can be specified here in key="value" format. [global_tags] @@ -614,18 +615,29 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false +{% if TRUE_CLUSTER %} + {% if grains.role == 'so-manager' %} +[[inputs.elasticsearch]] + servers = ["https://{{ MANAGER }}:9200"] + insecure_skip_verify = true + local = false + cluster_health = true + cluster_stats = true + {% endif %} + +{% else %} # # Read stats from one or more Elasticsearch servers or clusters -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} + {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] servers = ["https://{{ MANAGER }}:9200"] insecure_skip_verify = true -{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} + {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] insecure_skip_verify = true + {% endif %} {% endif %} - # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" @@ -673,11 +685,32 @@ # ## Commands array -{% if grains['role'] in ['so-manager', 'so-managersearch'] %} +{% if grains['role'] in ['so-manager'] %} [[inputs.exec]] commands = [ "/scripts/redis.sh", "/scripts/influxdbsize.sh", + "/scripts/raid.sh", + "/scripts/beatseps.sh" + ] + data_format = "influx" + ## Timeout for each command to complete. + timeout = "15s" +{% elif grains['role'] in ['so-managersearch'] %} +[[inputs.exec]] + commands = [ + "/scripts/redis.sh", + "/scripts/influxdbsize.sh", + "/scripts/eps.sh", + "/scripts/raid.sh", + "/scripts/beatseps.sh" + ] + data_format = "influx" + ## Timeout for each command to complete. + timeout = "15s" +{% elif grains['role'] in ['so-node'] %} +[[inputs.exec]] + commands = [ "/scripts/eps.sh", "/scripts/raid.sh", "/scripts/beatseps.sh" diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index faba0fabc..1226c42e4 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -15,37 +15,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=beatseps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/beatseps.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" -FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/beatseps.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" + FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" + + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" - fi -exit 0 diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index c84b6bec9..12cf3ece6 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=checkfiles -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) +if [ ! $THEGREP ]; then -echo "faffiles files=$FILES" + FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) + + echo "faffiles files=$FILES" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index dcc4b9051..99f001552 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -15,36 +15,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=eps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/eps.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/eps.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi + + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "consumptioneps eps=${EVENTS%%.*}" + +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "esteps eps=${EVENTS%%.*}" - fi -exit 0 diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index be5aaa1d2..7922a7ab6 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -15,35 +15,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=helixeps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/helixevents.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/helixevents.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" + + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "helixeps eps=${EVENTS%%.*}" +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "helixeps eps=${EVENTS%%.*}" - -fi - -exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/influxdbsize.sh b/salt/telegraf/scripts/influxdbsize.sh index 9bab7815b..f2ed41f35 100644 --- a/salt/telegraf/scripts/influxdbsize.sh +++ b/salt/telegraf/scripts/influxdbsize.sh @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=influxsize -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) +if [ ! $THEGREP ]; then -echo "influxsize kbytes=$INFLUXSIZE" + INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) + + echo "influxsize kbytes=$INFLUXSIZE" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index 0557137e7..d43f16d14 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,18 +15,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=oldpcap -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -# Get the data -OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) -DATE=$(date +%s) -AGE=$(($DATE - $OLDPCAP)) +if [ ! $THEGREP ]; then -echo "pcapage seconds=$AGE" + # Get the data + OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + DATE=$(date +%s) + AGE=$(($DATE - $OLDPCAP)) + + echo "pcapage seconds=$AGE" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index 0938bb658..c0aabe75f 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -15,19 +15,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=raid -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf -RAIDLOG=/var/log/raid/status.log -RAIDSTATUS=$(cat /var/log/raid/status.log) +THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ -f "$RAIDLOG" ]; then - echo "raid $RAIDSTATUS" +if [ ! $THEGREP ]; then + + if [ -f "$RAIDLOG" ]; then + echo "raid $RAIDSTATUS" + else + exit 0 + fi else exit 0 fi diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index 04079c63b..613e6cdea 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -16,16 +16,14 @@ # along with this program. If not, see . -APP=redis -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') -PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') +if [ ! $THEGREP ]; then -echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED" + UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') + PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') + + echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED" +else + exit 0 +fi diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh index 23096d903..1baf38d2b 100644 --- a/salt/telegraf/scripts/sostatus.sh +++ b/salt/telegraf/scripts/sostatus.sh @@ -14,20 +14,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +THEGREP=$(ps -ef | grep $0 | grep -v grep) -APP=sostatus -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf -SOSTATUSLOG=/var/log/sostatus/status.log -SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) +if [ ! $THEGREP ]; then -if [ -f "$SOSTATUSLOG" ]; then - echo "sostatus status=$SOSTATUSSTATUS" -else + SOSTATUSLOG=/var/log/sostatus/status.log + SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) + + if [ -f "$SOSTATUSLOG" ]; then + echo "sostatus status=$SOSTATUSSTATUS" + else + exit 0 + fi +else exit 0 fi diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index ad88ccc8d..a5c974a73 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -15,31 +15,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=stenoloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -TSFILE=/var/log/telegraf/laststenodrop.log -if [ -f "$TSFILE" ]; then - LASTTS=$(cat $TSFILE) +if [ ! $THEGREP ]; then + + TSFILE=/var/log/telegraf/laststenodrop.log + if [ -f "$TSFILE" ]; then + LASTTS=$(cat $TSFILE) + else + LASTTS=0 + fi + + # Get the data + LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) + CURRENTTS=$(echo $LOGLINE | awk '{print $1}') + + if [[ "$CURRENTTS" != "$LASTTS" ]]; then + DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') + echo $CURRENTTS > $TSFILE + else + DROP=0 + fi + + echo "stenodrop drop=$DROP" else - LASTTS=0 -fi - -# Get the data -LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) -CURRENTTS=$(echo $LOGLINE | awk '{print $1}') - -if [[ "$CURRENTTS" != "$LASTTS" ]]; then - DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') - echo $CURRENTTS > $TSFILE -else - DROP=0 -fi - -echo "stenodrop drop=$DROP" \ No newline at end of file + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 08f8c23eb..0ae867b29 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -16,37 +16,32 @@ # along with this program. If not, see . -APP=suriloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) -CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) +if [ ! $THEGREP ]; then -if [ $CHECKIT == 2 ]; then - declare RESULT=($SURILOG) + SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) + CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) - CURRENTDROP=${RESULT[4]} - PASTDROP=${RESULT[14]} - DROPPED=$((CURRENTDROP - PASTDROP)) - if [ $DROPPED == 0 ]; then - LOSS=0 - echo "suridrop drop=0" - else - CURRENTPACKETS=${RESULT[9]} - PASTPACKETS=${RESULT[19]} - TOTALCURRENT=$((CURRENTPACKETS + CURRENTDROP)) - TOTALPAST=$((PASTPACKETS + PASTDROP)) - TOTAL=$((TOTALCURRENT - TOTALPAST)) + if [ $CHECKIT == 2 ]; then + declare RESULT=($SURILOG) - LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) - echo "suridrop drop=$LOSS" - fi + CURRENTDROP=${RESULT[4]} + PASTDROP=${RESULT[14]} + DROPPED=$((CURRENTDROP - PASTDROP)) + if [ $DROPPED == 0 ]; then + LOSS=0 + echo "suridrop drop=0" + else + CURRENTPACKETS=${RESULT[9]} + PASTPACKETS=${RESULT[19]} + TOTALCURRENT=$((CURRENTPACKETS + CURRENTDROP)) + TOTALPAST=$((PASTPACKETS + PASTDROP)) + TOTAL=$((TOTALCURRENT - TOTALPAST)) + + LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) + echo "suridrop drop=$LOSS" + fi else echo "suridrop drop=0" fi \ No newline at end of file diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index aa8a222a3..995971b18 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -18,35 +18,32 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp -APP=zeekcaploss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} - ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log -elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then - WORKERS=1 - ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log -else - echo 'Zeek capture_loss.log not found' >/dev/stderr - exit 2 -fi +if [ ! $THEGREP ]; then -LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt -if [ -f "$ZEEKLOG" ]; then - CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g') - if [ -f "$LASTCAPTURELOSSLOG" ]; then - LASTTS=$(cat $LASTCAPTURELOSSLOG) - if [[ "$LASTTS" != "$CURRENTTS" ]]; then - LOSS=$(tail -$WORKERS $ZEEKLOG | awk -F, '{print $NF}' | sed 's/}//' | awk -v WORKERS=$WORKERS -F: '{LOSS += $2 / WORKERS} END { print LOSS}') - echo "zeekcaptureloss loss=$LOSS" + if [ -d "/host/nsm/zeek/spool/logger" ]; then + WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} + ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log + elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then + WORKERS=1 + ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log + else + echo 'Zeek capture_loss.log not found' >/dev/stderr + exit 2 fi - fi - echo "$CURRENTTS" > $LASTCAPTURELOSSLOG + + LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt + if [ -f "$ZEEKLOG" ]; then + CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g') + if [ -f "$LASTCAPTURELOSSLOG" ]; then + LASTTS=$(cat $LASTCAPTURELOSSLOG) + if [[ "$LASTTS" != "$CURRENTTS" ]]; then + LOSS=$(tail -$WORKERS $ZEEKLOG | awk -F, '{print $NF}' | sed 's/}//' | awk -v WORKERS=$WORKERS -F: '{LOSS += $2 / WORKERS} END { print LOSS}') + echo "zeekcaptureloss loss=$LOSS" + fi + fi + echo "$CURRENTTS" > $LASTCAPTURELOSSLOG +else + exit 0 fi diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index 0c1a714ba..559c6b15f 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -17,34 +17,31 @@ # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage -APP=zeekloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) -declare RESULT=($ZEEKLOG) -CURRENTDROP=${RESULT[3]} -# zeek likely not running if this is true -if [[ $CURRENTDROP == "rcvd:" ]]; then - CURRENTDROP=0 - PASTDROP=0 - DROPPED=0 +if [ ! $THEGREP ]; then + + ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) + declare RESULT=($ZEEKLOG) + CURRENTDROP=${RESULT[3]} + # zeek likely not running if this is true + if [[ $CURRENTDROP == "rcvd:" ]]; then + CURRENTDROP=0 + PASTDROP=0 + DROPPED=0 + else + PASTDROP=${RESULT[9]} + DROPPED=$((CURRENTDROP - PASTDROP)) + fi + if [[ "$DROPPED" -le 0 ]]; then + LOSS=0 + echo "zeekdrop drop=0" + else + CURRENTPACKETS=${RESULT[5]} + PASTPACKETS=${RESULT[11]} + TOTAL=$((CURRENTPACKETS - PASTPACKETS)) + LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) + echo "zeekdrop drop=$LOSS" else - PASTDROP=${RESULT[9]} - DROPPED=$((CURRENTDROP - PASTDROP)) -fi -if [[ "$DROPPED" -le 0 ]]; then - LOSS=0 - echo "zeekdrop drop=0" -else - CURRENTPACKETS=${RESULT[5]} - PASTPACKETS=${RESULT[11]} - TOTAL=$((CURRENTPACKETS - PASTPACKETS)) - LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) - echo "zeekdrop drop=$LOSS" + exit 0 fi \ No newline at end of file From f5b04117728780797bdd1d7c50c9375fd6f10f27 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:11:19 -0400 Subject: [PATCH 184/188] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 1226c42e4..50cfeff91 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -40,6 +40,7 @@ if [ ! $THEGREP ]; then fi echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + fi else exit 0 From 54322f5e9d608d0f8feade99bd8b23ee29322a50 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:17:02 -0400 Subject: [PATCH 185/188] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 50cfeff91..22189e43a 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -17,7 +17,7 @@ THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/beatseps.txt' EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" From 0c0edbaac8168c240937571459a8fda19b73bccb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:29:46 -0400 Subject: [PATCH 186/188] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 2 +- salt/telegraf/scripts/checkfiles.sh | 4 ++-- salt/telegraf/scripts/eps.sh | 4 ++-- salt/telegraf/scripts/helixeps.sh | 4 ++-- salt/telegraf/scripts/influxdbsize.sh | 4 ++-- salt/telegraf/scripts/oldpcap.sh | 4 ++-- salt/telegraf/scripts/raid.sh | 4 ++-- salt/telegraf/scripts/redis.sh | 5 ++--- salt/telegraf/scripts/sostatus.sh | 4 ++-- salt/telegraf/scripts/stenoloss.sh | 4 ++-- salt/telegraf/scripts/suriloss.sh | 4 ++-- salt/telegraf/scripts/zeekcaptureloss.sh | 4 ++-- salt/telegraf/scripts/zeekloss.sh | 4 ++-- 13 files changed, 25 insertions(+), 26 deletions(-) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 22189e43a..5e8256c22 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 12cf3ece6..1d7a44382 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 99f001552..25332e94a 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/eps.txt' EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index 7922a7ab6..d24f1d1e7 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/helixevents.txt' EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" diff --git a/salt/telegraf/scripts/influxdbsize.sh b/salt/telegraf/scripts/influxdbsize.sh index f2ed41f35..46e230a8a 100644 --- a/salt/telegraf/scripts/influxdbsize.sh +++ b/salt/telegraf/scripts/influxdbsize.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index d43f16d14..f23c0c83f 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then # Get the data OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index c0aabe75f..03e309c38 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then if [ -f "$RAIDLOG" ]; then echo "raid $RAIDSTATUS" diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index 613e6cdea..b448bba2d 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -15,10 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -THEGREP=$(ps -ef | grep $0 | grep -v grep) - -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh index 1baf38d2b..a7222b67d 100644 --- a/salt/telegraf/scripts/sostatus.sh +++ b/salt/telegraf/scripts/sostatus.sh @@ -14,9 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then SOSTATUSLOG=/var/log/sostatus/status.log SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index a5c974a73..028637e16 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then TSFILE=/var/log/telegraf/laststenodrop.log if [ -f "$TSFILE" ]; then diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 0ae867b29..1f43fbaf8 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -16,9 +16,9 @@ # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 995971b18..03c9188ea 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -18,9 +18,9 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index 559c6b15f..c9bc843cf 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -17,9 +17,9 @@ # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) declare RESULT=($ZEEKLOG) From be6933e8fb80536be70bbf12697bbfeb958c6085 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 14:20:00 -0400 Subject: [PATCH 187/188] Change EPS for Telegraf --- salt/telegraf/scripts/checkfiles.sh | 2 +- salt/telegraf/scripts/eps.sh | 2 +- salt/telegraf/scripts/helixeps.sh | 1 + salt/telegraf/scripts/suriloss.sh | 1 + salt/telegraf/scripts/zeekcaptureloss.sh | 1 + salt/telegraf/scripts/zeekloss.sh | 1 + 6 files changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 1d7a44382..5c6ab56c1 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -19,7 +19,7 @@ THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then - FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) + FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l) echo "faffiles files=$FILES" else diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 25332e94a..b497c2519 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -39,7 +39,7 @@ if [ ! "$THEGREP" ]; then fi echo "consumptioneps eps=${EVENTS%%.*}" - + fi else exit 0 fi diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index d24f1d1e7..1411cc40b 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -38,6 +38,7 @@ if [ ! "$THEGREP" ]; then fi echo "helixeps eps=${EVENTS%%.*}" + fi else exit 0 fi \ No newline at end of file diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 1f43fbaf8..2d0a56106 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -42,6 +42,7 @@ if [ ! "$THEGREP" ]; then LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) echo "suridrop drop=$LOSS" fi + fi else echo "suridrop drop=0" fi \ No newline at end of file diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 03c9188ea..6cb2dd2e2 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -44,6 +44,7 @@ if [ ! "$THEGREP" ]; then fi fi echo "$CURRENTTS" > $LASTCAPTURELOSSLOG + fi else exit 0 fi diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index c9bc843cf..3dbd42833 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -42,6 +42,7 @@ if [ ! "$THEGREP" ]; then TOTAL=$((CURRENTPACKETS - PASTPACKETS)) LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) echo "zeekdrop drop=$LOSS" + fi else exit 0 fi \ No newline at end of file From 5072c24134da383deda3c4dff8540fe71913f1f4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 20 Apr 2021 08:12:44 -0400 Subject: [PATCH 188/188] Adjust sostatus timers --- salt/common/init.sls | 2 +- salt/telegraf/etc/telegraf.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 37d6b6768..6ef841ea4 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -242,7 +242,7 @@ sostatus_log: '/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1': cron.present: - user: root - - minute: '*/5' + - minute: '*/1' - hour: '*' - daymonth: '*' - month: '*' diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index dac9bf60e..a7be4d8a2 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -681,7 +681,7 @@ ] data_format = "influx" timeout = "15s" - interval = "180s" + interval = "60s" # ## Commands array