From ce8db8abdb09c9c1ed1df922bc2e96ec57536655 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Feb 2021 10:51:04 -0500 Subject: [PATCH 01/20] [fix] Only run salt commands during reinstall if master is configured --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index e7112163e..b00c066d8 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1790,7 +1790,7 @@ reinstall_init() { local service_retry_count=20 { - if command -v salt-call &> /dev/null; then + if command -v salt-call &> /dev/null && grep -q "master:" /etc/salt/minion; then # Disable schedule so highstate doesn't start running during the install salt-call -l info schedule.disable From d9b4c09cf06924bbe43b46a82810544862958fcf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Feb 2021 10:52:18 -0500 Subject: [PATCH 02/20] [fix] Don't show irrelevant prompts during airgap setup --- setup/so-setup | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index beb9b40a3..5f0501dce 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -379,6 +379,10 @@ if [[ $is_import ]]; then PLAYBOOK=0 fi +if [[ $is_airgap ]]; then + PATCHSCHEDULENAME=manual +fi + # Start user prompts if [[ $is_helix ]]; then @@ -393,7 +397,7 @@ if [[ $is_helix || $is_sensor || $is_import ]]; then calculate_useable_cores fi -if [[ ! $is_import ]]; then +if [[ ! $is_airgap && ! $is_import ]]; then collect_patch_schedule fi @@ -446,7 +450,7 @@ if [[ $is_manager || $is_import ]]; then get_redirect fi -if [[ $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ]]; then +if [[ ! $is_airgap && $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ]]; then whiptail_manager_updates if [[ $setup_type == 'network' && $MANAGERUPDATES == 1 ]]; then whiptail_manager_updates_warning From 188d844d2724188706dc9678d1d9d42db3dedff3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Feb 2021 13:49:39 -0500 Subject: [PATCH 03/20] Redirect stderr of minion grep to /dev/null --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index b00c066d8..6745884ea 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1790,7 +1790,7 @@ reinstall_init() { local service_retry_count=20 { - if command -v salt-call &> /dev/null && grep -q "master:" /etc/salt/minion; then + if command -v salt-call &> /dev/null && grep -q "master:" /etc/salt/minion 2> /dev/null; then # Disable schedule so highstate doesn't start running during the install salt-call -l info schedule.disable From 1ffa7afefa10d325c4e5e40e87c7b056858c6e77 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 11 Feb 2021 16:20:29 -0500 Subject: [PATCH 04/20] eval-net answerfile corrections * HOSTNAME: standalone -> eval * install_type: STANDALONE -> EVAL --- setup/automation/eval-net-centos | 2 +- setup/automation/eval-net-ubuntu | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/automation/eval-net-centos b/setup/automation/eval-net-centos index f40ecc28e..abd0c4765 100644 --- a/setup/automation/eval-net-centos +++ b/setup/automation/eval-net-centos @@ -34,7 +34,7 @@ GRAFANA=1 # HELIXAPIKEY= HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit -HOSTNAME=standalone +HOSTNAME=eval install_type=EVAL # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= diff --git a/setup/automation/eval-net-ubuntu b/setup/automation/eval-net-ubuntu index fafb98cd4..132b8766e 100644 --- a/setup/automation/eval-net-ubuntu +++ b/setup/automation/eval-net-ubuntu @@ -34,8 +34,8 @@ GRAFANA=1 # HELIXAPIKEY= HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit -HOSTNAME=standalone -install_type=STANDALONE +HOSTNAME=eval +install_type=EVAL # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= # LSPIPELINEBATCH= From 8323f3f57aece95f3ae80464a56eede7e1930922 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 12 Feb 2021 12:23:45 -0500 Subject: [PATCH 05/20] [fix] Fix logic to correctly hide prompt --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 5f0501dce..05fb22bd7 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -450,7 +450,7 @@ if [[ $is_manager || $is_import ]]; then get_redirect fi -if [[ ! $is_airgap && $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ]]; then +if [[ ! $is_airgap && ( $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ) ]]; then whiptail_manager_updates if [[ $setup_type == 'network' && $MANAGERUPDATES == 1 ]]; then whiptail_manager_updates_warning From 724f5cad785ddb679725954eeff2e079d2621bee Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 12 Feb 2021 12:55:55 -0500 Subject: [PATCH 06/20] Warn user if using "securityonion" as hostname --- setup/so-functions | 7 +++++++ setup/so-whiptail | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 6745884ea..68aa7c76d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -398,6 +398,13 @@ collect_hostname() { whiptail_set_hostname "$HOSTNAME" + + if [[ $HOSTNAME == 'securityonion' ]]; then # Will only check HOSTNAME=securityonion once + if ! (whiptail_avoid_default_hostname); then + whiptail_set_hostname + fi + fi + while ! valid_hostname "$HOSTNAME"; do whiptail_invalid_hostname whiptail_set_hostname "$HOSTNAME" diff --git a/setup/so-whiptail b/setup/so-whiptail index c471fa47d..d87723826 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -28,6 +28,14 @@ whiptail_airgap() { whiptail_check_exitstatus $exitstatus } +whiptail_avoid_default_hostname() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" \ + --yesno "We suggest avoiding the default hostname of 'securityonion' in a distributed environment." 8 75 \ + --yes-button "Use" --no-button "Change" --defaultno +} + whiptail_basic_suri() { [ -n "$TESTING" ] && return From 4affb20b27ffaae3aaf259e348264b575c2b32b5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 12 Feb 2021 13:42:14 -0500 Subject: [PATCH 07/20] Give context to metadata tool choice --- setup/so-whiptail | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index d87723826..160fc3a8b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -945,13 +945,25 @@ whiptail_metadata_tool() { [ -n "$TESTING" ] && return + read -r -d '' message <<- EOM + What tool would you like to use to generate metadata? + + This question is asking specifically about metadata, which would be things like the connection log, DNS log, HTTP log, etc. This does not include NIDS alerts. + + If you choose Zeek for metadata, Suricata will still run to generate NIDS alerts. + + If you choose Suricata for metadata, it will generate NIDS alerts and metadata and Zeek will not run at all. + EOM + # Legacy variable naming - ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 \ - "ZEEK" "Zeek (formerly known as Bro)" ON \ - "SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3) + ZEEKVERSION=$(whiptail --title "Security Onion Setup" --menu "$message" 20 75 2 \ + "Zeek " "Use Zeek (Bro) for metadata and Suricata for NIDS alerts" \ + "Suricata " "Use Suricata for both metadata and NIDS alerts" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus + + ZEEKVERSION=$(echo "${ZEEKVERSION^^}" | tr -d ' ') } whiptail_nids() { From 5e8d09be510e989686f1ca396eae0d10cca85fc1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Feb 2021 09:42:35 -0500 Subject: [PATCH 08/20] [fix] Fix indent --- setup/so-functions | 102 ++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 68aa7c76d..bf4d80b7a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1494,50 +1494,50 @@ manager_global() { fi if [ -z "$DOCKERNET" ]; then - DOCKERNET=172.17.0.0 - DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 + DOCKERNET=172.17.0.0 + DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 else - DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 + DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 fi # Create a global file for global values - printf '%s\n'\ - "global:"\ - " soversion: '$SOVERSION'"\ - " hnmanager: '$HNMANAGER'"\ - " ntpserver: '$NTPSERVER'"\ - " dockernet: '$DOCKERNET'"\ - " proxy: '$PROXY'"\ - " mdengine: '$ZEEKVERSION'"\ - " ids: '$NIDS'"\ - " url_base: '$REDIRECTIT'"\ - " managerip: '$MAINIP'" > "$global_pillar" - - if [[ $is_airgap ]]; then - printf '%s\n'\ - " airgap: True"\ >> "$global_pillar" - else - printf '%s\n'\ - " airgap: False"\ >> "$global_pillar" - fi + printf '%s\n'\ + "global:"\ + " soversion: '$SOVERSION'"\ + " hnmanager: '$HNMANAGER'"\ + " ntpserver: '$NTPSERVER'"\ + " dockernet: '$DOCKERNET'"\ + " proxy: '$PROXY'"\ + " mdengine: '$ZEEKVERSION'"\ + " ids: '$NIDS'"\ + " url_base: '$REDIRECTIT'"\ + " managerip: '$MAINIP'" > "$global_pillar" + + if [[ $is_airgap ]]; then + printf '%s\n'\ + " airgap: True"\ >> "$global_pillar" + else + printf '%s\n'\ + " airgap: False"\ >> "$global_pillar" + fi - # Check if TheHive is enabled. If so, add creds and other details - if [[ "$THEHIVE" == "1" ]]; then - printf '%s\n'\ - " hiveuser: '$WEBUSER'"\ - " hivepassword: '$WEBPASSWD1'"\ - " hivekey: '$HIVEKEY'"\ - " hiveplaysecret: '$HIVEPLAYSECRET'"\ - " cortexuser: '$WEBUSER'"\ - " cortexpassword: '$WEBPASSWD1'"\ - " cortexkey: '$CORTEXKEY'"\ - " cortexorgname: 'SecurityOnion'"\ - " cortexorguser: 'soadmin'"\ - " cortexorguserkey: '$CORTEXORGUSERKEY'"\ - " cortexplaysecret: '$CORTEXPLAYSECRET'" >> "$global_pillar" - fi + # Check if TheHive is enabled. If so, add creds and other details + if [[ "$THEHIVE" == "1" ]]; then + printf '%s\n'\ + " hiveuser: '$WEBUSER'"\ + " hivepassword: '$WEBPASSWD1'"\ + " hivekey: '$HIVEKEY'"\ + " hiveplaysecret: '$HIVEPLAYSECRET'"\ + " cortexuser: '$WEBUSER'"\ + " cortexpassword: '$WEBPASSWD1'"\ + " cortexkey: '$CORTEXKEY'"\ + " cortexorgname: 'SecurityOnion'"\ + " cortexorguser: 'soadmin'"\ + " cortexorguserkey: '$CORTEXORGUSERKEY'"\ + " cortexplaysecret: '$CORTEXPLAYSECRET'" >> "$global_pillar" + fi - # Continue adding other details + # Continue adding other details printf '%s\n'\ " fleet_custom_hostname: "\ " fleet_manager: False"\ @@ -1548,7 +1548,7 @@ manager_global() { " fleet_ip: 'N/A'"\ " sensoronikey: '$SENSORONIKEY'"\ " wazuh: $WAZUH"\ - " managerupdate: $MANAGERUPDATES"\ + " managerupdate: $MANAGERUPDATES"\ " imagerepo: '$IMAGEREPO'"\ " pipeline: 'redis'"\ "sensoroni:"\ @@ -1565,21 +1565,21 @@ manager_global() { "elasticsearch:"\ " replicas: 0" >> "$global_pillar" if [ -n "$ESCLUSTERNAME" ]; then - printf '%s\n'\ - " true_cluster: True"\ - " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" + printf '%s\n'\ + " true_cluster: True"\ + " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" else printf '%s\n'\ - " true_cluster: False"\ - " true_cluster_name: 'so'" >> "$global_pillar" + " true_cluster: False"\ + " true_cluster_name: 'so'" >> "$global_pillar" fi printf '%s\n'\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\ - " cluster_routing_allocation_disk_watermark_low: '95%'"\ - " cluster_routing_allocation_disk_watermark_high: '98%'"\ - " cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\ + " cluster_routing_allocation_disk_watermark_low: '95%'"\ + " cluster_routing_allocation_disk_watermark_high: '98%'"\ + " cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\ " index_settings:"\ " so-beats:"\ " shards: 1"\ @@ -1640,10 +1640,10 @@ manager_global() { " upload_queue_size: 4"\ " encoding: 'gzip'"\ " interval: 5"\ - "backup:"\ - " locations:"\ - " - /opt/so/saltstack/local"\ - "soctopus:"\ + "backup:"\ + " locations:"\ + " - /opt/so/saltstack/local"\ + "soctopus:"\ " playbook:"\ " rulesets:"\ " - windows"\ From 23221065eb74925d0ea708687bbfaa566002500b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Feb 2021 09:43:54 -0500 Subject: [PATCH 09/20] Preset MANAGERUPDATES var for airgap since we don't prompt now --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index 05fb22bd7..548d9e7f4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -381,6 +381,7 @@ fi if [[ $is_airgap ]]; then PATCHSCHEDULENAME=manual + MANAGERUPDATES=0 fi # Start user prompts From 38a5b86813fdd406c2f50345f89aae394f2e2e79 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Feb 2021 10:24:07 -0500 Subject: [PATCH 10/20] Make apt-get syntax consistent --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index bf4d80b7a..6c57f3851 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -972,13 +972,13 @@ installer_prereq_packages() { echo "Installing required packages to run installer..." # Install network manager so we can do interface stuff if ! command -v nmcli > /dev/null 2>&1; then - retry 50 10 "apt-get install -y network-manager" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || exit 1 { systemctl enable NetworkManager systemctl start NetworkManager } >> "$setup_log" 2<&1 fi - retry 50 10 "apt-get install -y bc curl" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install bc curl" >> "$setup_log" 2>&1 || exit 1 fi } From 7a9f801eb1ef77008a1fd146a8ed826959e96b99 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Feb 2021 10:24:58 -0500 Subject: [PATCH 11/20] [fix] Add more `apt-get update` commands Fixes #2962 --- setup/so-functions | 3 +++ setup/so-setup | 1 + 2 files changed, 4 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 6c57f3851..4d5e85273 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -970,6 +970,7 @@ installer_prereq_packages() { elif [ "$OS" == ubuntu ]; then # Print message to stdout so the user knows setup is doing something echo "Installing required packages to run installer..." + retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 # Install network manager so we can do interface stuff if ! command -v nmcli > /dev/null 2>&1; then retry 50 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || exit 1 @@ -1961,6 +1962,8 @@ saltify() { exit 1 fi + retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 + if [ $OSVER != "xenial" ]; then # Switch to Python 3 as default if this is not xenial update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1 diff --git a/setup/so-setup b/setup/so-setup index 548d9e7f4..691e70709 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -130,6 +130,7 @@ if [[ -f automation/$automation && $(basename $automation) == $automation ]]; th if [[ ! $is_iso ]]; then echo "Installing sshpass for automated testing." >> $setup_log 2>&1 if [ "$OS" == ubuntu ]; then + retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y install sshpass" >> $setup_log 2>&1 || exit 1 else yum -y install sshpass >> $setup_log 2>&1 From 3323e900efe269d20d42a8480fcccd26d45af933 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Feb 2021 11:17:36 -0500 Subject: [PATCH 12/20] [fix] Fix indent (pt 2) --- setup/so-functions | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4d5e85273..9926f63c8 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1565,16 +1565,16 @@ manager_global() { " features: False"\ "elasticsearch:"\ " replicas: 0" >> "$global_pillar" - if [ -n "$ESCLUSTERNAME" ]; then - printf '%s\n'\ - " true_cluster: True"\ - " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" - else - printf '%s\n'\ - " true_cluster: False"\ - " true_cluster_name: 'so'" >> "$global_pillar" - fi + if [ -n "$ESCLUSTERNAME" ]; then printf '%s\n'\ + " true_cluster: True"\ + " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" + else + printf '%s\n'\ + " true_cluster: False"\ + " true_cluster_name: 'so'" >> "$global_pillar" + fi + printf '%s\n'\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\ From 0bd5ddf6a659930c2b0cff921f311f4c0f10e6b3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 16 Feb 2021 14:17:41 -0500 Subject: [PATCH 13/20] Grammar + misc fixes per PR review * Remove unnecessary `apt-get update` commands * Change `if ! (command); then exit 1; fi` to `command || exit 1` to avoid subshell --- setup/so-functions | 6 +----- setup/so-setup | 1 - setup/so-whiptail | 12 +++++++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 9926f63c8..33ac11a7f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1958,11 +1958,7 @@ saltify() { } >> "$setup_log" 2>&1 yum versionlock salt* else - if ! (DEBIAN_FRONTEND=noninteractive retry 50 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1); then - exit 1 - fi - - retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 + DEBIAN_FRONTEND=noninteractive retry 50 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1 if [ $OSVER != "xenial" ]; then # Switch to Python 3 as default if this is not xenial diff --git a/setup/so-setup b/setup/so-setup index 691e70709..548d9e7f4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -130,7 +130,6 @@ if [[ -f automation/$automation && $(basename $automation) == $automation ]]; th if [[ ! $is_iso ]]; then echo "Installing sshpass for automated testing." >> $setup_log 2>&1 if [ "$OS" == ubuntu ]; then - retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y install sshpass" >> $setup_log 2>&1 || exit 1 else yum -y install sshpass >> $setup_log 2>&1 diff --git a/setup/so-whiptail b/setup/so-whiptail index 160fc3a8b..eef2aba66 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -31,9 +31,15 @@ whiptail_airgap() { whiptail_avoid_default_hostname() { [ -n "$TESTING" ] && return + read -r -d '' message <<- EOM + To prevent hostname conflicts, avoid using the default 'securityonion' hostname in a distributed environment. + + You can choose to use this default hostname anyway, or change it to a new hostname. + EOM + whiptail --title "Security Onion Setup" \ - --yesno "We suggest avoiding the default hostname of 'securityonion' in a distributed environment." 8 75 \ - --yes-button "Use" --no-button "Change" --defaultno + --yesno "$message" 11 75 \ + --yes-button "Use Anyway" --no-button "Change" --defaultno } whiptail_basic_suri() { @@ -952,7 +958,7 @@ whiptail_metadata_tool() { If you choose Zeek for metadata, Suricata will still run to generate NIDS alerts. - If you choose Suricata for metadata, it will generate NIDS alerts and metadata and Zeek will not run at all. + If you choose Suricata for metadata, it will generate NIDS alerts and metadata, and Zeek will not run at all. EOM # Legacy variable naming From e6ae1af85f6905d900590775e84afd9dfab5ede7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 17 Feb 2021 10:47:06 -0500 Subject: [PATCH 14/20] test rotating strelka log at 100k --- salt/common/files/sensor-rotate.conf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/common/files/sensor-rotate.conf b/salt/common/files/sensor-rotate.conf index a7e4cb303..0529939d9 100644 --- a/salt/common/files/sensor-rotate.conf +++ b/salt/common/files/sensor-rotate.conf @@ -8,3 +8,17 @@ sharedscripts endscript } + +/nsm/strelka/log/strelka.log +{ + #daily + size 100k + rotate 14 + missingok + copytruncate + compress + create + extension .log + dateext + dateyesterday +} \ No newline at end of file From 2b07d89b5abdc794099160fcc682b2b6ba09fa6d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 17 Feb 2021 11:01:18 -0500 Subject: [PATCH 15/20] error: /opt/so/conf/sensor-rotate.conf:8 unknown option 'endscript' -- ignoring line --- salt/common/files/sensor-rotate.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/files/sensor-rotate.conf b/salt/common/files/sensor-rotate.conf index 0529939d9..ba23ce16f 100644 --- a/salt/common/files/sensor-rotate.conf +++ b/salt/common/files/sensor-rotate.conf @@ -6,7 +6,6 @@ nocompress create sharedscripts - endscript } /nsm/strelka/log/strelka.log From 7a595df5b6193912f90e3f364c6fc20ef3458fe0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 17 Feb 2021 11:17:41 -0500 Subject: [PATCH 16/20] strelka logrotate - https://github.com/Security-Onion-Solutions/securityonion/issues/2736 --- salt/common/files/sensor-rotate.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/files/sensor-rotate.conf b/salt/common/files/sensor-rotate.conf index ba23ce16f..cefd3944e 100644 --- a/salt/common/files/sensor-rotate.conf +++ b/salt/common/files/sensor-rotate.conf @@ -10,8 +10,7 @@ /nsm/strelka/log/strelka.log { - #daily - size 100k + daily rotate 14 missingok copytruncate From 0bef8b66620918bc01f5843b8c2211d1f0f9ee0b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 Feb 2021 09:26:11 -0500 Subject: [PATCH 17/20] limit number of eve.json files for suricata https://github.com/Security-Onion-Solutions/securityonion/issues/2989 --- salt/suricata/cron/so-suricata-eve-clean | 35 ++++++++++++++++++++++++ salt/suricata/init.sls | 20 ++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 salt/suricata/cron/so-suricata-eve-clean diff --git a/salt/suricata/cron/so-suricata-eve-clean b/salt/suricata/cron/so-suricata-eve-clean new file mode 100644 index 000000000..b931eeea2 --- /dev/null +++ b/salt/suricata/cron/so-suricata-eve-clean @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see .. /usr/sbin/so-common + +APP=so-suricata-eve-clean +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +MAXEVES={% salt['pillar.get']('suricata:cleanup:eve_json', 20) %} + +# Find eve files and remove them +NUMEVES=$(find /nsm/suricata/ -type f -name "eve-*.json" | wc -l) +while [ "$NUMEVES" -gt "$MAXEVES" ]; do + OLDESTEVE=$(find /nsm/suricata/ -type f -name "eve-*.json" -type f -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}') + rm -f $OLDESTEVE + NUMEVES=$(find /nsm/suricata/ -type f -name "eve-*.json" | wc -l) +done \ No newline at end of file diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 3de6e3568..708976cd9 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -177,6 +177,26 @@ disable_so-suricata_so-status.conf: - month: '*' - dayweek: '*' +so-suricata-eve-clean: + file.managed: + - name: /usr/sbin/so-suricata-eve-clean + - user: root + - group: root + - file_mode: 755 + - template: jinja + - source: salt://suricata/cron/so-suricata-eve-clean + +# Add eve clean cron +clean_suricata_eve_files: + cron.present: + - name: /usr/sbin/so-suricata-eve-clean > /dev/null 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% else %} {{sls}}_state_not_allowed: From 041d193f2d52a1655f6f1b12ec54b696b68c48fe Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 Feb 2021 09:37:37 -0500 Subject: [PATCH 18/20] fix brackets --- salt/suricata/cron/so-suricata-eve-clean | 2 +- salt/suricata/init.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/cron/so-suricata-eve-clean b/salt/suricata/cron/so-suricata-eve-clean index b931eeea2..dcaef3109 100644 --- a/salt/suricata/cron/so-suricata-eve-clean +++ b/salt/suricata/cron/so-suricata-eve-clean @@ -24,7 +24,7 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -MAXEVES={% salt['pillar.get']('suricata:cleanup:eve_json', 20) %} +MAXEVES={{ salt['pillar.get']('suricata:cleanup:eve_json', 20) }} # Find eve files and remove them NUMEVES=$(find /nsm/suricata/ -type f -name "eve-*.json" | wc -l) diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 708976cd9..a77248ce1 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -182,7 +182,7 @@ so-suricata-eve-clean: - name: /usr/sbin/so-suricata-eve-clean - user: root - group: root - - file_mode: 755 + - mode: 755 - template: jinja - source: salt://suricata/cron/so-suricata-eve-clean From 4b07d5e457b2bb559922d044acdcdaaa27bb5059 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 Feb 2021 09:39:54 -0500 Subject: [PATCH 19/20] add identifier to eve clean cron --- salt/suricata/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index a77248ce1..1a01d5e3a 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -190,6 +190,7 @@ so-suricata-eve-clean: clean_suricata_eve_files: cron.present: - name: /usr/sbin/so-suricata-eve-clean > /dev/null 2>&1 + - identifier: clean_suricata_eve_files - user: root - minute: '*/5' - hour: '*' From 74ca4487de3698cab7b4338571a7bc042d0e0e64 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 Feb 2021 09:51:40 -0500 Subject: [PATCH 20/20] ensure at least 2 eve files are kept https://github.com/Security-Onion-Solutions/securityonion/issues/2989 --- salt/suricata/cron/so-suricata-eve-clean | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/suricata/cron/so-suricata-eve-clean b/salt/suricata/cron/so-suricata-eve-clean index dcaef3109..a3cd6bc79 100644 --- a/salt/suricata/cron/so-suricata-eve-clean +++ b/salt/suricata/cron/so-suricata-eve-clean @@ -25,6 +25,9 @@ read lastPID < $lf echo $$ > $lf MAXEVES={{ salt['pillar.get']('suricata:cleanup:eve_json', 20) }} +if [ "$MAXEVES" -lt 2 ]; then + MAXEVES=2 +fi # Find eve files and remove them NUMEVES=$(find /nsm/suricata/ -type f -name "eve-*.json" | wc -l)