From 66ca7b266cd0a6d550112d421ab22d24161e6c99 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Aug 2020 14:44:23 -0400 Subject: [PATCH 01/38] first commit of importpcap node mode code, kek --- salt/firewall/assigned_hostgroups.map.yaml | 48 +++++++ salt/importpcap/bond.sls | 5 + setup/so-functions | 142 +++++++++++---------- setup/so-setup | 76 +++++++++-- 4 files changed, 193 insertions(+), 78 deletions(-) create mode 100644 salt/importpcap/bond.sls diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 2500c604a..eaafd45ca 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -480,3 +480,51 @@ role: localhost: portgroups: - {{ portgroups.all }} + importpcap: + chain: + DOCKER-USER: + hostgroups: + manager: + portgroups: + - {{ portgroups.kibana }} + - {{ portgroups.redis }} + - {{ portgroups.influxdb }} + - {{ portgroups.elasticsearch_rest }} + - {{ portgroups.elasticsearch_node }} + sensor: + portgroups: + - {{ portgroups.beats_5044 }} + - {{ portgroups.beats_5644 }} + search_node: + portgroups: + - {{ portgroups.redis }} + - {{ portgroups.elasticsearch_node }} + self: + portgroups: + - {{ portgroups.syslog}} + beats_endpoint: + portgroups: + - {{ portgroups.beats_5044 }} + beats_endpoint_ssl: + portgroups: + - {{ portgroups.beats_5644 }} + elasticsearch_rest: + portgroups: + - {{ portgroups.elasticsearch_rest }} + analyst: + portgroups: + - {{ portgroups.nginx }} + INPUT: + hostgroups: + anywhere: + portgroups: + - {{ portgroups.ssh }} + dockernet: + portgroups: + - {{ portgroups.all }} + localhost: + portgroups: + - {{ portgroups.all }} + minion: + portgroups: + - {{ portgroups.salt_manager }} \ No newline at end of file diff --git a/salt/importpcap/bond.sls b/salt/importpcap/bond.sls new file mode 100644 index 000000000..85a4065a2 --- /dev/null +++ b/salt/importpcap/bond.sls @@ -0,0 +1,5 @@ +configure_bond0: + network.managed: + - name: bond0 + - type: bond + - enabled: True \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index cb9c75437..12f8d2ec0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -142,65 +142,7 @@ secrets_pillar(){ fi } -# Enable Bro Logs -zeek_logs_enabled() { - echo "Enabling Bro Logs" >> "$setup_log" 2>&1 - local zeeklogs_pillar=./pillar/zeeklogs.sls - - printf '%s\n'\ - "zeeklogs:"\ - " enabled:" > "$zeeklogs_pillar" - - if [ "$MANAGERADV" = 'ADVANCED' ]; then - for BLOG in "${BLOGS[@]}"; do - echo " - $BLOG" | tr -d '"' >> "$zeeklogs_pillar" - done - else - printf '%s\n'\ - " - conn"\ - " - dce_rpc"\ - " - dhcp"\ - " - dhcpv6"\ - " - dnp3"\ - " - dns"\ - " - dpd"\ - " - files"\ - " - ftp"\ - " - http"\ - " - intel"\ - " - irc"\ - " - kerberos"\ - " - modbus"\ - " - mqtt"\ - " - notice"\ - " - ntlm"\ - " - openvpn"\ - " - pe"\ - " - radius"\ - " - rfb"\ - " - rdp"\ - " - signatures"\ - " - sip"\ - " - smb_files"\ - " - smb_mapping"\ - " - smtp"\ - " - snmp"\ - " - software"\ - " - ssh"\ - " - ssl"\ - " - syslog"\ - " - telnet"\ - " - tunnel"\ - " - weird"\ - " - mysql"\ - " - socks"\ - " - x509" >> "$zeeklogs_pillar" - fi - - printf '%s\n' '----' >> "$setup_log" 2>&1 - cat "$zeeklogs_pillar" >> "$setup_log" 2>&1 -} check_admin_pass() { check_pass_match "$ADMINPASS1" "$ADMINPASS2" "APMATCH" @@ -416,11 +358,19 @@ check_requirements() { req_cores=4 if [[ "$node_type" == 'sensor' ]]; then req_nics=2; else req_nics=1; fi if [[ "$node_type" == 'fleet' ]]; then req_mem=4; fi + elif [[ "$standalone_or_dist" == 'importpcap' ]]; then + req_mem=4 + req_cores=2 + req_nics=1 fi - if [[ $setup_type == 'network' ]]; then + if [[ $setup_type == 'network' ]] ; then if [[ -n $nsm_mount ]]; then - req_storage=100 + if [[ "$standalone_or_dist" == 'importpcap' ]]; then + req_storage=50 + else + req_storage=100 + fi if (( $(echo "$free_space_root < $req_storage" | bc -l) )); then whiptail_storage_requirements "/" "${free_space_root} GB" "${req_storage} GB" fi @@ -428,7 +378,11 @@ check_requirements() { whiptail_storage_requirements "/nsm" "${free_space_nsm} GB" "${req_storage} GB" fi else - req_storage=200 + if [[ "$standalone_or_dist" == 'importpcap' ]]; then + req_storage=50 + else + req_storage=200 + fi if (( $(echo "$free_space_root < $req_storage" | bc -l) )); then whiptail_storage_requirements "/" "${free_space_root} GB" "${req_storage} GB" fi @@ -720,7 +674,7 @@ docker_install() { else case "$install_type" in - 'MANAGER' | 'EVAL') + 'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORTPCAP') apt-get update >> "$setup_log" 2>&1 ;; *) @@ -1264,7 +1218,7 @@ saltify() { set_progress_str 6 'Installing various dependencies' yum -y install wget nmap-ncat >> "$setup_log" 2>&1 case "$install_type" in - 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE') + 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORTPCAP') reserve_group_ids >> "$setup_log" 2>&1 yum -y install epel-release >> "$setup_log" 2>&1 yum -y install sqlite argon2 curl mariadb-devel >> "$setup_log" 2>&1 @@ -1335,7 +1289,7 @@ saltify() { 'FLEET') if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi ;; - 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE') # TODO: should this also be HELIXSENSOR? + 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') # TODO: should this also be HELIXSENSOR? # Add saltstack repo(s) wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3001/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 @@ -1802,3 +1756,63 @@ es_heapsize() { export NODE_ES_HEAP_SIZE fi } + +# Enable Bro Logs +zeek_logs_enabled() { + echo "Enabling Bro Logs" >> "$setup_log" 2>&1 + + local zeeklogs_pillar=./pillar/zeeklogs.sls + + printf '%s\n'\ + "zeeklogs:"\ + " enabled:" > "$zeeklogs_pillar" + + if [ "$MANAGERADV" = 'ADVANCED' ]; then + for BLOG in "${BLOGS[@]}"; do + echo " - $BLOG" | tr -d '"' >> "$zeeklogs_pillar" + done + else + printf '%s\n'\ + " - conn"\ + " - dce_rpc"\ + " - dhcp"\ + " - dhcpv6"\ + " - dnp3"\ + " - dns"\ + " - dpd"\ + " - files"\ + " - ftp"\ + " - http"\ + " - intel"\ + " - irc"\ + " - kerberos"\ + " - modbus"\ + " - mqtt"\ + " - notice"\ + " - ntlm"\ + " - openvpn"\ + " - pe"\ + " - radius"\ + " - rfb"\ + " - rdp"\ + " - signatures"\ + " - sip"\ + " - smb_files"\ + " - smb_mapping"\ + " - smtp"\ + " - snmp"\ + " - software"\ + " - ssh"\ + " - ssl"\ + " - syslog"\ + " - telnet"\ + " - tunnel"\ + " - weird"\ + " - mysql"\ + " - socks"\ + " - x509" >> "$zeeklogs_pillar" + fi + + printf '%s\n' '----' >> "$setup_log" 2>&1 + cat "$zeeklogs_pillar" >> "$setup_log" 2>&1 +} \ No newline at end of file diff --git a/setup/so-setup b/setup/so-setup index 68ca99824..260642415 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -165,6 +165,8 @@ elif [ "$install_type" = 'FLEET' ]; then OSQUERY=1 elif [ "$install_type" = 'HELIXSENSOR' ]; then is_helix=true +elif [ "$install_type" = 'IMPORTPCAP' ]; then + is_importpcap=true fi if [[ $is_manager && $is_sensor ]]; then @@ -173,8 +175,10 @@ elif [[ $is_fleet_standalone ]]; then check_requirements "dist" "fleet" elif [[ $is_sensor && ! $is_eval ]]; then check_requirements "dist" "sensor" -elif [[ $is_distmanager || $is_minion ]]; then +elif [[ $is_distmanager || $is_minion ]] && [[ ! $is_importpcap ]]; then check_requirements "dist" +elif [[ $is_importpcap ]]; then + check_requirements "importpcap" fi whiptail_patch_schedule @@ -239,13 +243,38 @@ if [[ $is_node ]]; then CURCLOSEDAYS=30 fi +if [[ $is_importpcap ]]; then + patch_schedule=Automatic + RULESETUP=ETOPEN + NSMSETUP=BASIC + HNSENSOR=inherit + MANAGERUPDATES=0 + MANAGERADV=BASIC + ZEEKVERSION=ZEEK + NIDS=Suricata + RULESETUP=ETOPEN + GRAFANA=0 + OSQUERY=0 + WAZUH=0 + THEHIVE=0 + PLAYBOOK=0 + STRELKA=0 + +fi + + # Start user prompts + + if [[ $is_helix || $is_sensor ]]; then whiptail_sensor_nics +fi + +if [[ $is_helix || $is_sensor || $is_importpcap ]]; then calculate_useable_cores fi -if [[ $is_helix || $is_manager ]]; then +if [[ $is_helix || $is_manager || $is_importpcap ]]; then whiptail_homenet_manager fi @@ -274,6 +303,9 @@ if [[ $is_manager ]]; then if [[ $STRELKA == 1 ]]; then whiptail_strelka_rules fi +fi + +if [[ $is_manager || $is_importpcap ]]; then collect_webuser_inputs get_redirect fi @@ -335,7 +367,7 @@ else FLEETNODEPASSWD1=$WEBPASSWD1 fi -if [[ $is_manager ]]; then whiptail_so_allow; fi +if [[ $is_manager || $is_importpcap ]]; then whiptail_so_allow; fi whiptail_make_changes @@ -359,7 +391,7 @@ fi } >> $setup_log 2>&1 -if [[ $is_manager ]]; then +if [[ $is_manager || $is_importpcap ]]; then { generate_passwords; secrets_pillar; @@ -399,6 +431,9 @@ fi if [[ $is_sensor || $is_helix ]]; then set_progress_str 3 'Configuring sensor interface' configure_network_sensor >> $setup_log 2>&1 + fi + + if [[ $is_sensor || $is_helix || $is_importpcap ]]; then set_progress_str 4 'Generating sensor pillar' sensor_pillar >> $setup_log 2>&1 fi @@ -415,7 +450,7 @@ fi set_progress_str 9 'Initializing Salt minion' configure_minion "$minion_type" >> $setup_log 2>&1 - if [[ $is_manager || $is_helix ]]; then + if [[ $is_manager || $is_helix || $is_importpcap ]]; then set_progress_str 10 'Configuring Salt master' { create_local_directories; @@ -459,7 +494,7 @@ fi accept_salt_key_remote >> $setup_log 2>&1 fi - if [[ $is_manager ]]; then + if [[ $is_manager || $is_importpcap ]]; then set_progress_str 20 'Accepting Salt key' salt-key -ya "$MINION_ID" >> $setup_log 2>&1 fi @@ -472,10 +507,15 @@ fi salt-call state.apply salt.minion -l info >> $setup_log 2>&1 fi + if [[ $is_importpcap ]]; then + set_progress_str 22 'Configuring bond interface' + salt-call state.apply importpcap.bond -l info >> $setup_log 2>&1 + fi + set_progress_str 23 'Generating CA and checking in' salt_checkin >> $setup_log 2>&1 - if [[ $is_manager || $is_helix ]]; then + if [[ $is_manager || $is_helix || $is_importpcap ]]; then set_progress_str 25 'Configuring firewall' set_initial_firewall_policy >> $setup_log 2>&1 @@ -485,14 +525,18 @@ fi set_progress_str 26 'Downloading containers from the internet' fi - salt-call state.apply -l info registry >> $setup_log 2>&1 - docker_seed_registry 2>> "$setup_log" # ~ 60% when finished + if [[ ! $is_importpcap ]]; then + salt-call state.apply -l info registry >> $setup_log 2>&1 + docker_seed_registry 2>> "$setup_log" # ~ 60% when finished + fi set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 - set_progress_str 61 "$(print_salt_state_apply 'idstools')" - salt-call state.apply -l info idstools >> $setup_log 2>&1 + if [[ ! $is_importpcap ]]; then + set_progress_str 61 "$(print_salt_state_apply 'idstools')" + salt-call state.apply -l info idstools >> $setup_log 2>&1 + fi set_progress_str 61 "$(print_salt_state_apply 'suricata.manager')" salt-call state.apply -l info suricata.manager >> $setup_log 2>&1 @@ -513,7 +557,7 @@ fi set_progress_str 64 "$(print_salt_state_apply 'nginx')" salt-call state.apply -l info nginx >> $setup_log 2>&1 - if [[ $is_manager || $is_node ]]; then + if [[ $is_manager || $is_node || $is_importpcap ]]; then set_progress_str 64 "$(print_salt_state_apply 'elasticsearch')" salt-call state.apply -l info elasticsearch >> $setup_log 2>&1 fi @@ -521,7 +565,9 @@ fi if [[ $is_sensor ]]; then set_progress_str 65 "$(print_salt_state_apply 'pcap')" salt-call state.apply -l info pcap >> $setup_log 2>&1 + fi + if [[ $is_sensor || $is_importpcap ]]; then set_progress_str 66 "$(print_salt_state_apply 'suricata')" salt-call state.apply -l info suricata >> $setup_log 2>&1 @@ -534,13 +580,15 @@ fi salt-call state.apply -l info curator >> $setup_log 2>&1 fi - if [[ $is_manager ]]; then + if [[ $is_manager || $is_importpcap ]]; then set_progress_str 69 "$(print_salt_state_apply 'soc')" salt-call state.apply -l info soc >> $setup_log 2>&1 set_progress_str 70 "$(print_salt_state_apply 'kibana')" salt-call state.apply -l info kibana >> $setup_log 2>&1 + fi + if [[ $is_manager ]]; then set_progress_str 71 "$(print_salt_state_apply 'elastalert')" salt-call state.apply -l info elastalert >> $setup_log 2>&1 @@ -598,7 +646,7 @@ fi fi fi - if [[ $is_manager || $is_helix ]]; then + if [[ $is_manager || $is_helix || $is_importpcap ]]; then set_progress_str 81 "$(print_salt_state_apply 'utility')" salt-call state.apply -l info utility >> $setup_log 2>&1 fi From 83dc35c72082d64f7abae925b79619d9a7f06d84 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Aug 2020 15:24:11 -0400 Subject: [PATCH 02/38] add importpcap mode to whiptail --- setup/so-whiptail | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index 5b201818e..2ba6da10e 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -475,6 +475,7 @@ whiptail_install_type() { "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ "DISTRIBUTED" "Distributed install submenu " OFF \ + "IMPORTPCAP" "Import PCAP mode " OFF \ 3>&1 1>&2 2>&3 ) From 8079dc54fc81380a1fae899809fe77a0bd53140a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Aug 2020 15:42:22 -0400 Subject: [PATCH 03/38] add stuff for /etc/salt/minion to get populated for importpcap node --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 12f8d2ec0..b7050cf70 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -302,7 +302,7 @@ configure_minion() { 'helix') echo "master: $HOSTNAME" >> "$minion_config" ;; - 'manager' | 'eval' | 'managersearch' | 'standalone') + 'manager' | 'eval' | 'managersearch' | 'standalone' | 'importpcap') printf '%s\n'\ "master: $HOSTNAME"\ "mysql.host: '$MAINIP'"\ @@ -856,7 +856,7 @@ got_root() { get_minion_type() { local minion_type case "$install_type" in - 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET' | 'STANDALONE') + 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET' | 'STANDALONE' | 'IMPORTPCAP') minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]') ;; 'HELIXSENSOR') From d7801acea5453b77b5e071348b8a8065e3c6c7f5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Aug 2020 17:09:41 -0400 Subject: [PATCH 04/38] add mode 1 --- salt/importpcap/bond.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/importpcap/bond.sls b/salt/importpcap/bond.sls index 85a4065a2..14de298ec 100644 --- a/salt/importpcap/bond.sls +++ b/salt/importpcap/bond.sls @@ -2,4 +2,5 @@ configure_bond0: network.managed: - name: bond0 - type: bond + - mode: '1' - enabled: True \ No newline at end of file From d668b850336574fd48c834618f4cc8d9687998da Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 11:09:12 -0400 Subject: [PATCH 05/38] copy_ssh_key for is_importpcap also --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index c14b71c1e..9a9c2788e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -410,7 +410,7 @@ fi host_pillar >> $setup_log 2>&1 -if [[ $is_minion ]]; then +if [[ $is_minion || $is_importpcap ]]; then set_updates >> $setup_log 2>&1 copy_ssh_key >> $setup_log 2>&1 fi From 2c6a20fee98fd079e4bd694544f9f857c3c63ae9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 11:11:21 -0400 Subject: [PATCH 06/38] enlarge whiptail for install type selection --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 2ba6da10e..92c130f3d 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -471,7 +471,7 @@ whiptail_install_type() { # What kind of install are we doing? install_type=$(whiptail --title "Security Onion Setup" --radiolist \ - "Choose install type:" 10 65 3 \ + "Choose install type:" 12 65 3 \ "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ "DISTRIBUTED" "Distributed install submenu " OFF \ From 24b77fa855c8f797a6d5323d2a5f2dffc8afade1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 11:16:52 -0400 Subject: [PATCH 07/38] enlarge whiptail for install type selection --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 92c130f3d..264390d30 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -471,7 +471,7 @@ whiptail_install_type() { # What kind of install are we doing? install_type=$(whiptail --title "Security Onion Setup" --radiolist \ - "Choose install type:" 12 65 3 \ + "Choose install type:" 10 65 4 \ "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ "DISTRIBUTED" "Distributed install submenu " OFF \ From 7d11fc345f14a2c15a5e0d15d496b9ec6445b0dd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 11:19:31 -0400 Subject: [PATCH 08/38] dont ask for patch schedule for importpcap node --- setup/so-setup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 9a9c2788e..9c0255cea 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -181,7 +181,9 @@ elif [[ $is_importpcap ]]; then check_requirements "importpcap" fi -whiptail_patch_schedule +if [[ ! $is_importpcap ]]; then + whiptail_patch_schedule +fi case "$setup_type" in 'iso') From 2d7aefed0d96eb8b6d88eb3dd6e343050181ffd0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 11:42:48 -0400 Subject: [PATCH 09/38] add IMPORTPCAP node to set_hostname --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index eb93e2975..d2e2be748 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1521,7 +1521,7 @@ set_hostname() { set_hostname_iso - if [[ ! $install_type =~ ^(MANAGER|EVAL|HELIXSENSOR|MANAGERSEARCH|STANDALONE)$ ]]; then + if [[ ! $install_type =~ ^(MANAGER|EVAL|HELIXSENSOR|MANAGERSEARCH|STANDALONE|IMPORTPCAP)$ ]]; then if ! getent hosts "$MSRV"; then echo "$MSRVIP $MSRV" >> /etc/hosts fi From 7c3070655b53416c921756afaa050bab8238ecae Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 13:39:17 -0400 Subject: [PATCH 10/38] copy_minion_tmp_files for IMPORTPCAP too --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d2e2be748..c10cc6661 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -417,7 +417,7 @@ copy_salt_master_config() { copy_minion_tmp_files() { case "$install_type" in - 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE') + 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') echo "Copying pillar and salt files in $temp_install_dir to $local_salt_dir" cp -Rv "$temp_install_dir"/pillar/ $local_salt_dir/ >> "$setup_log" 2>&1 if [ -d "$temp_install_dir"/salt ] ; then From fadd81c9f38d807f7595c14267549bbe9a8b3884 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 13:58:29 -0400 Subject: [PATCH 11/38] so-importpcap to ssl state --- salt/ssl/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index d7c84675e..fdb40a0bf 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -7,7 +7,7 @@ {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} -{% if grains.id.split('_')|last in ['manager', 'eval', 'standalone'] %} +{% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'importpcap'] %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} @@ -72,7 +72,7 @@ influxkeyperms: - mode: 640 - group: 939 -{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone'] %} +{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-importpcap'] %} /etc/pki/filebeat.key: x509.private_key_managed: @@ -287,7 +287,7 @@ fleetkeyperms: - group: 939 {% endif %} -{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone'] %} +{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-importpcap'] %} fbcertdir: file.directory: From a8147d7d3baf9ee1abbd2d029c2e13bf4589e743 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 14:19:58 -0400 Subject: [PATCH 12/38] add importpcap to salt_checkin for setup ssl/ca --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index c10cc6661..800d57a90 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1357,7 +1357,7 @@ saltify() { salt_checkin() { case "$install_type" in - 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE') # Fix Mine usage + 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') # Fix Mine usage { echo "Building Certificate Authority"; salt-call state.apply ca; From 9649994f734cf7bb97349b724588baf43e82ab5c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 14:40:02 -0400 Subject: [PATCH 13/38] add importpcap to pillar/top --- pillar/top.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index c11b66eaa..153945163 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -80,3 +80,10 @@ base: - logstash.search - elasticsearch.search - minions.{{ grains.id }} + + '*_importpcap': + - zeeklogs + - secrets + - elasticsearch.eval + - global + - minions.{{ grains.id }} \ No newline at end of file From 86b118ba1a0eacc1d07d59fb6b4defd5e802bb1c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 15:00:32 -0400 Subject: [PATCH 14/38] add importpcap to local assigned hostgroups yaml --- files/firewall/assigned_hostgroups.local.map.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/files/firewall/assigned_hostgroups.local.map.yaml b/files/firewall/assigned_hostgroups.local.map.yaml index 5d9b662b6..b39d34ae7 100644 --- a/files/firewall/assigned_hostgroups.local.map.yaml +++ b/files/firewall/assigned_hostgroups.local.map.yaml @@ -13,6 +13,7 @@ role: fleet: heavynode: helixsensor: + importpcap: manager: managersearch: standalone: From d7b55c110935bf143ec1d4d0a1a1505cc0741501 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 15:21:07 -0400 Subject: [PATCH 15/38] add so-status map for importpcap --- salt/common/maps/importpcap.map.jinja | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 salt/common/maps/importpcap.map.jinja diff --git a/salt/common/maps/importpcap.map.jinja b/salt/common/maps/importpcap.map.jinja new file mode 100644 index 000000000..f412a030e --- /dev/null +++ b/salt/common/maps/importpcap.map.jinja @@ -0,0 +1,14 @@ +{% set docker = { + 'containers': [ + 'so-filebeat', + 'so-nginx', + 'so-soc', + 'so-kratos', + 'so-elasticsearch', + 'so-kibana', + 'so-suricata', + 'so-zeek', + 'so-soctopus', + 'so-sensoroni' + ] +} %} \ No newline at end of file From 7933bafd5524ae79bb5cd9aa6a4ebd0feb6c20fc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 15:46:45 -0400 Subject: [PATCH 16/38] more fixes for importpcap node --- pillar/top.sls | 2 +- salt/elasticsearch/init.sls | 2 +- salt/nginx/etc/nginx.conf.so-importpcap | 326 ++++++++++++++++++++++++ 3 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 salt/nginx/etc/nginx.conf.so-importpcap diff --git a/pillar/top.sls b/pillar/top.sls index 153945163..44f56edc5 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,7 +2,7 @@ base: '*': - patch.needs_restarting - '*_eval or *_helix or *_heavynode or *_sensor or *_standalone': + '*_eval or *_helix or *_heavynode or *_sensor or *_standalone or *_importpcap': - match: compound - zeek diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index f3777481c..0b2090591 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -23,7 +23,7 @@ {% set FEATURES = '' %} {% endif %} -{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} +{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-importpcap'] %} {% set esclustername = salt['pillar.get']('manager:esclustername', '') %} {% set esheap = salt['pillar.get']('manager:esheap', '') %} {% elif grains['role'] in ['so-node','so-heavynode'] %} diff --git a/salt/nginx/etc/nginx.conf.so-importpcap b/salt/nginx/etc/nginx.conf.so-importpcap new file mode 100644 index 000000000..9c919c764 --- /dev/null +++ b/salt/nginx/etc/nginx.conf.so-importpcap @@ -0,0 +1,326 @@ +{%- set managerip = salt['pillar.get']('manager:mainip', '') %} +{%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager') %} +{%- set FLEET_NODE = salt['pillar.get']('global:fleet_node') %} +{%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', None) %} +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + client_max_body_size 1024M; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + #server { + # listen 80 default_server; + # listen [::]:80 default_server; + # server_name _; + # root /opt/socore/html; + # index index.html; + + # Load configuration files for the default server block. + #include /etc/nginx/default.d/*.conf; + + # location / { + # } + + # error_page 404 /404.html; + # location = /40x.html { + # } + + # error_page 500 502 503 504 /50x.html; + # location = /50x.html { + # } + #} + server { + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; + } + +{% if FLEET_MANAGER %} + server { + listen 8090 ssl http2 default_server; + server_name _; + root /opt/socore/html; + index blank.html; + + ssl_certificate "/etc/pki/nginx/server.crt"; + ssl_certificate_key "/etc/pki/nginx/server.key"; + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 10m; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + + location ~ ^/kolide.agent.Api/(RequestEnrollment|RequestConfig|RequestQueries|PublishLogs|PublishResults|CheckHealth)$ { + grpc_pass grpcs://{{ managerip }}:8080; + grpc_set_header Host $host; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_buffering off; + } + + } +{% endif %} + +# Settings for a TLS enabled server. + + server { + listen 443 ssl http2 default_server; + #listen [::]:443 ssl http2 default_server; + server_name _; + root /opt/socore/html; + index index.html; + + ssl_certificate "/etc/pki/nginx/server.crt"; + ssl_certificate_key "/etc/pki/nginx/server.key"; + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 10m; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + + # Load configuration files for the default server block. + #include /etc/nginx/default.d/*.conf; + + location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) { + proxy_pass http://{{ managerip }}:9822; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + auth_request /auth/sessions/whoami; + proxy_pass http://{{ managerip }}:9822/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location ~ ^/auth/.*?(whoami|login|logout|settings) { + rewrite /auth/(.*) /$1 break; + proxy_pass http://{{ managerip }}:4433; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /cyberchef/ { + auth_request /auth/sessions/whoami; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /navigator/ { + auth_request /auth/sessions/whoami; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /packages/ { + try_files $uri =206; + auth_request /auth/sessions/whoami; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /grafana/ { + auth_request /auth/sessions/whoami; + rewrite /grafana/(.*) /$1 break; + proxy_pass http://{{ managerip }}:3000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /kibana/ { + auth_request /auth/sessions/whoami; + rewrite /kibana/(.*) /$1 break; + proxy_pass http://{{ managerip }}:5601/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /nodered/ { + proxy_pass http://{{ managerip }}:1880/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /playbook/ { + proxy_pass http://{{ managerip }}:3200/playbook/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + {%- if FLEET_NODE %} + location /fleet/ { + return 301 https://{{ FLEET_IP }}/fleet; + } + {%- else %} + location /fleet/ { + proxy_pass https://{{ managerip }}:8080; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + {%- endif %} + + location /thehive/ { + proxy_pass http://{{ managerip }}:9000/thehive/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /cortex/ { + proxy_pass http://{{ managerip }}:9001/cortex/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /soctopus/ { + proxy_pass http://{{ managerip }}:7000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /kibana/app/soc/ { + rewrite ^/kibana/app/soc/(.*) /soc/$1 permanent; + } + + location /kibana/app/fleet/ { + rewrite ^/kibana/app/fleet/(.*) /fleet/$1 permanent; + } + + location /kibana/app/soctopus/ { + rewrite ^/kibana/app/soctopus/(.*) /soctopus/$1 permanent; + } + + location /sensoroniagents/ { + proxy_pass http://{{ managerip }}:9822/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + + error_page 401 = @error401; + + location @error401 { + add_header Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400"; + return 302 /auth/self-service/browser/flows/login; + } + + #error_page 404 /404.html; + # location = /usr/share/nginx/html/40x.html { + #} + + error_page 500 502 503 504 /50x.html; + location = /usr/share/nginx/html/50x.html { + } + } + +} From b02332d84aaef4be45aa0a1e00c4eee11dbd9a80 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 16:18:11 -0400 Subject: [PATCH 17/38] fix global pillar location for setup --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 800d57a90..26871658c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -962,7 +962,7 @@ manager_pillar() { } manager_global() { - local global_pillar="$local_salt_dir/pillar/global.sls" + local global_pillar="$temp_install_dir/pillar/global.sls" if [ -z "$SENSOR_CHECKIN_INTERVAL_MS" ]; then SENSOR_CHECKIN_INTERVAL_MS=10000 From 0c2ea53f259759c9b536e2a926ca298459d8b5f2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 Aug 2020 16:42:46 -0400 Subject: [PATCH 18/38] revert back to local_salt_dir --- setup/so-functions | 2 +- setup/so-setup | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 26871658c..800d57a90 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -962,7 +962,7 @@ manager_pillar() { } manager_global() { - local global_pillar="$temp_install_dir/pillar/global.sls" + local global_pillar="$local_salt_dir/pillar/global.sls" if [ -z "$SENSOR_CHECKIN_INTERVAL_MS" ]; then SENSOR_CHECKIN_INTERVAL_MS=10000 diff --git a/setup/so-setup b/setup/so-setup index 9c0255cea..cae5d1029 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -247,6 +247,7 @@ fi if [[ $is_importpcap ]]; then patch_schedule=Automatic + MTU=1500 RULESETUP=ETOPEN NSMSETUP=BASIC HNSENSOR=inherit From f6a85ac852cf69e39405a7208bb2f983046fa5f6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Aug 2020 12:27:21 -0400 Subject: [PATCH 19/38] top and seed registry for importpcap node --- salt/top.sls | 18 ++++++++++++++++++ setup/so-functions | 17 ++++++++++++++++- setup/so-setup | 8 ++++---- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index 34b825355..316523f08 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -386,3 +386,21 @@ base: - fleet - fleet.install_package - filebeat + + '*_importpcap and G@saltversion:{{saltversion}}': + - match: compound + - ca + - ssl + - registry + - manager + - common + - nginx + - soc + - firewall + - suricata.manager + - elasticsearch + - kibana + - suricata + - filebeat + - utility + - schedule diff --git a/setup/so-functions b/setup/so-functions index 5ae4b7716..9bc2bacfd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -718,6 +718,20 @@ docker_seed_registry() { local VERSION="$SOVERSION" if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then + if [ "$install_type" == 'IMPORTPCAP' ]; then + local TRUSTED_CONTAINERS=(\ + "so-nginx:$VERSION" \ + "so-filebeat:$VERSION" \ + "so-suricata:$VERSION" \ + "so-soc:$VERSION" \ + "so-elasticsearch:$VERSION" \ + "so-kibana:$VERSION" \ + "so-kratos:$VERSION" \ + "so-suricata:$VERSION" \ + "so-registry:$VERSION" \ + "so-zeek:$VERSION" + ) + else local TRUSTED_CONTAINERS=(\ "so-nginx:$VERSION" \ "so-filebeat:$VERSION" \ @@ -729,7 +743,8 @@ docker_seed_registry() { "so-telegraf:$VERSION" \ "so-zeek:$VERSION" ) - if [ "$install_type" != 'HELIXSENSOR' ]; then + fi + if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORTPCAP' ]; then TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \ "so-acng:$VERSION" \ "so-thehive-cortex:$VERSION" \ diff --git a/setup/so-setup b/setup/so-setup index cae5d1029..dd9e73b32 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -529,10 +529,10 @@ fi set_progress_str 26 'Downloading containers from the internet' fi - if [[ ! $is_importpcap ]]; then - salt-call state.apply -l info registry >> $setup_log 2>&1 - docker_seed_registry 2>> "$setup_log" # ~ 60% when finished - fi + + salt-call state.apply -l info registry >> $setup_log 2>&1 + docker_seed_registry 2>> "$setup_log" # ~ 60% when finished + set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 From ec62668eb74c5e9809b9a4b2ca812d4d2d837a66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Aug 2020 12:31:37 -0400 Subject: [PATCH 20/38] firewall rules for importpcap node --- salt/firewall/assigned_hostgroups.map.yaml | 3 +++ setup/so-functions | 2 +- setup/so-setup | 2 -- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 5cee13b10..fe7e12135 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -499,6 +499,9 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + minion: + portgroups: + - {{ portgroups.docker_registry }} sensor: portgroups: - {{ portgroups.beats_5044 }} diff --git a/setup/so-functions b/setup/so-functions index 9bc2bacfd..fd2e88516 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1568,7 +1568,7 @@ set_initial_firewall_policy() { $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost minion "$MAINIP" $default_salt_dir/pillar/data/addtotab.sh managertab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" ;; - 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE') + 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') $default_salt_dir/salt/common/tools/sbin/so-firewall includehost manager "$MAINIP" $default_salt_dir/salt/common/tools/sbin/so-firewall includehost minion "$MAINIP" $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensor "$MAINIP" diff --git a/setup/so-setup b/setup/so-setup index dd9e73b32..0994bad06 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -529,10 +529,8 @@ fi set_progress_str 26 'Downloading containers from the internet' fi - salt-call state.apply -l info registry >> $setup_log 2>&1 docker_seed_registry 2>> "$setup_log" # ~ 60% when finished - set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 From ee914504243295c7c4db26e49236391d339c74ec Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Aug 2020 13:30:41 -0400 Subject: [PATCH 21/38] fix patch schedule name for importpcap node --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 0994bad06..9a52cb64a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -246,7 +246,7 @@ if [[ $is_node ]]; then fi if [[ $is_importpcap ]]; then - patch_schedule=Automatic + PATCHSCHEDULENAME=Automatic MTU=1500 RULESETUP=ETOPEN NSMSETUP=BASIC From a81d14463cf46dd789abadbc3e3e8a3416906925 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Aug 2020 15:01:20 -0400 Subject: [PATCH 22/38] add logstash to registry for importpcap, change PATCHSCHEDULENAME=auto --- setup/so-functions | 1 + setup/so-setup | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index fd2e88516..d9f00d42c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -720,6 +720,7 @@ docker_seed_registry() { if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then if [ "$install_type" == 'IMPORTPCAP' ]; then local TRUSTED_CONTAINERS=(\ + "so-logstash:$VERSION" \ "so-nginx:$VERSION" \ "so-filebeat:$VERSION" \ "so-suricata:$VERSION" \ diff --git a/setup/so-setup b/setup/so-setup index 48153becb..3bdc82c0e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -244,7 +244,7 @@ if [[ $is_node ]]; then fi if [[ $is_importpcap ]]; then - PATCHSCHEDULENAME=Automatic + PATCHSCHEDULENAME=auto MTU=1500 RULESETUP=ETOPEN NSMSETUP=BASIC From de054032379a490f871f375e9e0f254067871516 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Aug 2020 15:52:15 -0400 Subject: [PATCH 23/38] ensure nids rules dir exists --- salt/suricata/manager.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/suricata/manager.sls b/salt/suricata/manager.sls index 5998a484b..e287069cb 100644 --- a/salt/suricata/manager.sls +++ b/salt/suricata/manager.sls @@ -1,3 +1,10 @@ +nidsrulesdir: + file.directory: + - name: /opt/so/rules/nids + - user: 939 + - group: 939 + - makedirs: True + surilocaldir: file.directory: - name: /opt/so/saltstack/local/salt/suricata From 53b4a73bb902f508d15baf3a7d1d5497b7e029a4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Aug 2020 15:59:08 -0400 Subject: [PATCH 24/38] add idstools to importpcap node --- salt/suricata/manager.sls | 7 ------- salt/top.sls | 1 + setup/so-setup | 6 ++---- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/salt/suricata/manager.sls b/salt/suricata/manager.sls index e287069cb..5998a484b 100644 --- a/salt/suricata/manager.sls +++ b/salt/suricata/manager.sls @@ -1,10 +1,3 @@ -nidsrulesdir: - file.directory: - - name: /opt/so/rules/nids - - user: 939 - - group: 939 - - makedirs: True - surilocaldir: file.directory: - name: /opt/so/saltstack/local/salt/suricata diff --git a/salt/top.sls b/salt/top.sls index 316523f08..509f6b2c2 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -397,6 +397,7 @@ base: - nginx - soc - firewall + - idstools - suricata.manager - elasticsearch - kibana diff --git a/setup/so-setup b/setup/so-setup index 3bdc82c0e..d9a13210c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -533,10 +533,8 @@ fi set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 - if [[ ! $is_importpcap ]]; then - set_progress_str 61 "$(print_salt_state_apply 'idstools')" - salt-call state.apply -l info idstools >> $setup_log 2>&1 - fi + set_progress_str 61 "$(print_salt_state_apply 'idstools')" + salt-call state.apply -l info idstools >> $setup_log 2>&1 set_progress_str 61 "$(print_salt_state_apply 'suricata.manager')" salt-call state.apply -l info suricata.manager >> $setup_log 2>&1 From 6260a0aeaaba73099fdafeb0978e92a761c7393d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Aug 2020 16:29:35 -0400 Subject: [PATCH 25/38] add idstools to docker registry for importpcap node --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d9f00d42c..dda15b65e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -720,7 +720,8 @@ docker_seed_registry() { if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then if [ "$install_type" == 'IMPORTPCAP' ]; then local TRUSTED_CONTAINERS=(\ - "so-logstash:$VERSION" \ + "so-logstash:$VERSION" \ + "so-idstools:$VERSION" \ "so-nginx:$VERSION" \ "so-filebeat:$VERSION" \ "so-suricata:$VERSION" \ From 5a0df2719311303b854fe9fe79afb5d21375422f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 10:27:15 -0400 Subject: [PATCH 26/38] rename importpcap node to import --- .../assigned_hostgroups.local.map.yaml | 2 +- pillar/top.sls | 4 +- ...{importpcap.map.jinja => import.map.jinja} | 4 +- salt/elasticsearch/init.sls | 2 +- salt/firewall/assigned_hostgroups.map.yaml | 2 +- salt/{importpcap => import}/bond.sls | 0 salt/ssl/init.sls | 6 +-- salt/top.sls | 2 +- setup/so-functions | 28 +++++------ setup/so-setup | 46 +++++++++---------- setup/so-whiptail | 2 +- 11 files changed, 48 insertions(+), 50 deletions(-) rename salt/common/maps/{importpcap.map.jinja => import.map.jinja} (74%) rename salt/{importpcap => import}/bond.sls (100%) diff --git a/files/firewall/assigned_hostgroups.local.map.yaml b/files/firewall/assigned_hostgroups.local.map.yaml index b39d34ae7..50ef751a4 100644 --- a/files/firewall/assigned_hostgroups.local.map.yaml +++ b/files/firewall/assigned_hostgroups.local.map.yaml @@ -13,7 +13,7 @@ role: fleet: heavynode: helixsensor: - importpcap: + import: manager: managersearch: standalone: diff --git a/pillar/top.sls b/pillar/top.sls index 44f56edc5..73d66ef2a 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,7 +2,7 @@ base: '*': - patch.needs_restarting - '*_eval or *_helix or *_heavynode or *_sensor or *_standalone or *_importpcap': + '*_eval or *_helix or *_heavynode or *_sensor or *_standalone or *_import': - match: compound - zeek @@ -81,7 +81,7 @@ base: - elasticsearch.search - minions.{{ grains.id }} - '*_importpcap': + '*_import': - zeeklogs - secrets - elasticsearch.eval diff --git a/salt/common/maps/importpcap.map.jinja b/salt/common/maps/import.map.jinja similarity index 74% rename from salt/common/maps/importpcap.map.jinja rename to salt/common/maps/import.map.jinja index f412a030e..adb266809 100644 --- a/salt/common/maps/importpcap.map.jinja +++ b/salt/common/maps/import.map.jinja @@ -7,8 +7,6 @@ 'so-elasticsearch', 'so-kibana', 'so-suricata', - 'so-zeek', - 'so-soctopus', - 'so-sensoroni' + 'so-zeek' ] } %} \ No newline at end of file diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 0f92a5d9c..48c4c99aa 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -23,7 +23,7 @@ {% set FEATURES = '' %} {% endif %} -{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-importpcap'] %} +{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %} {% set esclustername = salt['pillar.get']('manager:esclustername', '') %} {% set esheap = salt['pillar.get']('manager:esheap', '') %} {% set ismanager = True %} diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index fe7e12135..ef9e6fe0c 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -488,7 +488,7 @@ role: localhost: portgroups: - {{ portgroups.all }} - importpcap: + import: chain: DOCKER-USER: hostgroups: diff --git a/salt/importpcap/bond.sls b/salt/import/bond.sls similarity index 100% rename from salt/importpcap/bond.sls rename to salt/import/bond.sls diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index af9495e59..c0b48dd1f 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -7,7 +7,7 @@ {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} -{% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'importpcap'] %} +{% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'import'] %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} @@ -72,7 +72,7 @@ influxkeyperms: - mode: 640 - group: 939 -{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-importpcap'] %} +{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %} /etc/pki/filebeat.key: x509.private_key_managed: @@ -322,7 +322,7 @@ fleetkeyperms: - group: 939 {% endif %} -{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-importpcap'] %} +{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import'] %} fbcertdir: file.directory: diff --git a/salt/top.sls b/salt/top.sls index 509f6b2c2..a51978b1e 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -387,7 +387,7 @@ base: - fleet.install_package - filebeat - '*_importpcap and G@saltversion:{{saltversion}}': + '*_import and G@saltversion:{{saltversion}}': - match: compound - ca - ssl diff --git a/setup/so-functions b/setup/so-functions index dda15b65e..26d80c98a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -302,7 +302,7 @@ configure_minion() { 'helix') echo "master: $HOSTNAME" >> "$minion_config" ;; - 'manager' | 'eval' | 'managersearch' | 'standalone' | 'importpcap') + 'manager' | 'eval' | 'managersearch' | 'standalone' | 'import') printf '%s\n'\ "master: $HOSTNAME"\ "mysql.host: '$MAINIP'"\ @@ -358,7 +358,7 @@ check_requirements() { req_cores=4 if [[ "$node_type" == 'sensor' ]]; then req_nics=2; else req_nics=1; fi if [[ "$node_type" == 'fleet' ]]; then req_mem=4; fi - elif [[ "$standalone_or_dist" == 'importpcap' ]]; then + elif [[ "$standalone_or_dist" == 'import' ]]; then req_mem=4 req_cores=2 req_nics=1 @@ -366,7 +366,7 @@ check_requirements() { if [[ $setup_type == 'network' ]] ; then if [[ -n $nsm_mount ]]; then - if [[ "$standalone_or_dist" == 'importpcap' ]]; then + if [[ "$standalone_or_dist" == 'import' ]]; then req_storage=50 else req_storage=100 @@ -378,7 +378,7 @@ check_requirements() { whiptail_storage_requirements "/nsm" "${free_space_nsm} GB" "${req_storage} GB" fi else - if [[ "$standalone_or_dist" == 'importpcap' ]]; then + if [[ "$standalone_or_dist" == 'import' ]]; then req_storage=50 else req_storage=200 @@ -417,7 +417,7 @@ copy_salt_master_config() { copy_minion_tmp_files() { case "$install_type" in - 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') + 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') echo "Copying pillar and salt files in $temp_install_dir to $local_salt_dir" cp -Rv "$temp_install_dir"/pillar/ $local_salt_dir/ >> "$setup_log" 2>&1 if [ -d "$temp_install_dir"/salt ] ; then @@ -674,7 +674,7 @@ docker_install() { else case "$install_type" in - 'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORTPCAP') + 'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT') apt-get update >> "$setup_log" 2>&1 ;; *) @@ -718,7 +718,7 @@ docker_seed_registry() { local VERSION="$SOVERSION" if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then - if [ "$install_type" == 'IMPORTPCAP' ]; then + if [ "$install_type" == 'IMPORT' ]; then local TRUSTED_CONTAINERS=(\ "so-logstash:$VERSION" \ "so-idstools:$VERSION" \ @@ -746,7 +746,7 @@ docker_seed_registry() { "so-zeek:$VERSION" ) fi - if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORTPCAP' ]; then + if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORT' ]; then TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \ "so-acng:$VERSION" \ "so-thehive-cortex:$VERSION" \ @@ -874,7 +874,7 @@ got_root() { get_minion_type() { local minion_type case "$install_type" in - 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET' | 'STANDALONE' | 'IMPORTPCAP') + 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET' | 'STANDALONE' | 'IMPORT') minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]') ;; 'HELIXSENSOR') @@ -1246,7 +1246,7 @@ saltify() { set_progress_str 6 'Installing various dependencies' yum -y install wget nmap-ncat >> "$setup_log" 2>&1 case "$install_type" in - 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORTPCAP') + 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORT') reserve_group_ids >> "$setup_log" 2>&1 yum -y install epel-release >> "$setup_log" 2>&1 yum -y install sqlite argon2 curl mariadb-devel >> "$setup_log" 2>&1 @@ -1317,7 +1317,7 @@ saltify() { 'FLEET') if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi ;; - 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') # TODO: should this also be HELIXSENSOR? + 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') # TODO: should this also be HELIXSENSOR? # Add saltstack repo(s) wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3001.1/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 @@ -1374,7 +1374,7 @@ saltify() { salt_checkin() { case "$install_type" in - 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') # Fix Mine usage + 'MANAGER' | 'EVAL' | 'HELIXSENSOR' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') # Fix Mine usage { echo "Building Certificate Authority"; salt-call state.apply ca; @@ -1538,7 +1538,7 @@ set_hostname() { set_hostname_iso - if [[ ! $install_type =~ ^(MANAGER|EVAL|HELIXSENSOR|MANAGERSEARCH|STANDALONE|IMPORTPCAP)$ ]]; then + if [[ ! $install_type =~ ^(MANAGER|EVAL|HELIXSENSOR|MANAGERSEARCH|STANDALONE|IMPORT)$ ]]; then if ! getent hosts "$MSRV"; then echo "$MSRVIP $MSRV" >> /etc/hosts fi @@ -1570,7 +1570,7 @@ set_initial_firewall_policy() { $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost minion "$MAINIP" $default_salt_dir/pillar/data/addtotab.sh managertab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" ;; - 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORTPCAP') + 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') $default_salt_dir/salt/common/tools/sbin/so-firewall includehost manager "$MAINIP" $default_salt_dir/salt/common/tools/sbin/so-firewall includehost minion "$MAINIP" $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensor "$MAINIP" diff --git a/setup/so-setup b/setup/so-setup index d9a13210c..4a8cf259e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -163,8 +163,8 @@ elif [ "$install_type" = 'FLEET' ]; then OSQUERY=1 elif [ "$install_type" = 'HELIXSENSOR' ]; then is_helix=true -elif [ "$install_type" = 'IMPORTPCAP' ]; then - is_importpcap=true +elif [ "$install_type" = 'IMPORT' ]; then + is_import=true fi if [[ $is_manager && $is_sensor ]]; then @@ -173,13 +173,13 @@ elif [[ $is_fleet_standalone ]]; then check_requirements "dist" "fleet" elif [[ $is_sensor && ! $is_eval ]]; then check_requirements "dist" "sensor" -elif [[ $is_distmanager || $is_minion ]] && [[ ! $is_importpcap ]]; then +elif [[ $is_distmanager || $is_minion ]] && [[ ! $is_import ]]; then check_requirements "dist" -elif [[ $is_importpcap ]]; then - check_requirements "importpcap" +elif [[ $is_import ]]; then + check_requirements "import" fi -if [[ ! $is_importpcap ]]; then +if [[ ! $is_import ]]; then whiptail_patch_schedule fi @@ -243,7 +243,7 @@ if [[ $is_node ]]; then CURCLOSEDAYS=30 fi -if [[ $is_importpcap ]]; then +if [[ $is_import ]]; then PATCHSCHEDULENAME=auto MTU=1500 RULESETUP=ETOPEN @@ -271,11 +271,11 @@ if [[ $is_helix || $is_sensor ]]; then whiptail_sensor_nics fi -if [[ $is_helix || $is_sensor || $is_importpcap ]]; then +if [[ $is_helix || $is_sensor || $is_import ]]; then calculate_useable_cores fi -if [[ $is_helix || $is_manager || $is_importpcap ]]; then +if [[ $is_helix || $is_manager || $is_import ]]; then whiptail_homenet_manager fi @@ -306,7 +306,7 @@ if [[ $is_manager ]]; then fi fi -if [[ $is_manager || $is_importpcap ]]; then +if [[ $is_manager || $is_import ]]; then collect_webuser_inputs get_redirect fi @@ -368,7 +368,7 @@ else FLEETNODEPASSWD1=$WEBPASSWD1 fi -if [[ $is_manager || $is_importpcap ]]; then whiptail_so_allow; fi +if [[ $is_manager || $is_import ]]; then whiptail_so_allow; fi whiptail_make_changes @@ -392,7 +392,7 @@ fi } >> $setup_log 2>&1 -if [[ $is_manager || $is_importpcap ]]; then +if [[ $is_manager || $is_import ]]; then { generate_passwords; secrets_pillar; @@ -411,7 +411,7 @@ fi host_pillar >> $setup_log 2>&1 -if [[ $is_minion || $is_importpcap ]]; then +if [[ $is_minion || $is_import ]]; then set_updates >> $setup_log 2>&1 copy_ssh_key >> $setup_log 2>&1 fi @@ -434,7 +434,7 @@ fi configure_network_sensor >> $setup_log 2>&1 fi - if [[ $is_sensor || $is_helix || $is_importpcap ]]; then + if [[ $is_sensor || $is_helix || $is_import ]]; then set_progress_str 4 'Generating sensor pillar' sensor_pillar >> $setup_log 2>&1 fi @@ -451,7 +451,7 @@ fi set_progress_str 9 'Initializing Salt minion' configure_minion "$minion_type" >> $setup_log 2>&1 - if [[ $is_manager || $is_helix || $is_importpcap ]]; then + if [[ $is_manager || $is_helix || $is_import ]]; then set_progress_str 10 'Configuring Salt master' { create_local_directories; @@ -496,7 +496,7 @@ fi accept_salt_key_remote >> $setup_log 2>&1 fi - if [[ $is_manager || $is_importpcap ]]; then + if [[ $is_manager || $is_import ]]; then set_progress_str 20 'Accepting Salt key' salt-key -ya "$MINION_ID" >> $setup_log 2>&1 fi @@ -509,15 +509,15 @@ fi salt-call state.apply salt.minion -l info >> $setup_log 2>&1 fi - if [[ $is_importpcap ]]; then + if [[ $is_import ]]; then set_progress_str 22 'Configuring bond interface' - salt-call state.apply importpcap.bond -l info >> $setup_log 2>&1 + salt-call state.apply import.bond -l info >> $setup_log 2>&1 fi set_progress_str 23 'Generating CA and checking in' salt_checkin >> $setup_log 2>&1 - if [[ $is_manager || $is_helix || $is_importpcap ]]; then + if [[ $is_manager || $is_helix || $is_import ]]; then set_progress_str 25 'Configuring firewall' set_initial_firewall_policy >> $setup_log 2>&1 @@ -555,7 +555,7 @@ fi set_progress_str 64 "$(print_salt_state_apply 'nginx')" salt-call state.apply -l info nginx >> $setup_log 2>&1 - if [[ $is_manager || $is_node || $is_importpcap ]]; then + if [[ $is_manager || $is_node || $is_import ]]; then set_progress_str 64 "$(print_salt_state_apply 'elasticsearch')" salt-call state.apply -l info elasticsearch >> $setup_log 2>&1 fi @@ -565,7 +565,7 @@ fi salt-call state.apply -l info pcap >> $setup_log 2>&1 fi - if [[ $is_sensor || $is_importpcap ]]; then + if [[ $is_sensor || $is_import ]]; then set_progress_str 66 "$(print_salt_state_apply 'suricata')" salt-call state.apply -l info suricata >> $setup_log 2>&1 @@ -578,7 +578,7 @@ fi salt-call state.apply -l info curator >> $setup_log 2>&1 fi - if [[ $is_manager || $is_importpcap ]]; then + if [[ $is_manager || $is_import ]]; then set_progress_str 69 "$(print_salt_state_apply 'soc')" salt-call state.apply -l info soc >> $setup_log 2>&1 @@ -644,7 +644,7 @@ fi fi fi - if [[ $is_manager || $is_helix || $is_importpcap ]]; then + if [[ $is_manager || $is_helix || $is_import ]]; then set_progress_str 81 "$(print_salt_state_apply 'utility')" salt-call state.apply -l info utility >> $setup_log 2>&1 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 264390d30..f84090e4f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -475,7 +475,7 @@ whiptail_install_type() { "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ "DISTRIBUTED" "Distributed install submenu " OFF \ - "IMPORTPCAP" "Import PCAP mode " OFF \ + "IMPORT" "Standalone to import PCAP or log files " OFF \ 3>&1 1>&2 2>&3 ) From 0f53b4d7038a591a04e8e0b4809176e7c0c42541 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 10:39:31 -0400 Subject: [PATCH 27/38] set esheapsize and filebeat config for import node --- salt/filebeat/etc/filebeat.yml | 4 ++-- setup/so-setup | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 2b8a4118f..a4525b494 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -74,7 +74,7 @@ filebeat.modules: # List of prospectors to fetch data. filebeat.inputs: #------------------------------ Log prospector -------------------------------- -{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" or grains['role'] == "so-standalone" %} +{%- if grains['role'] in ['so-sensor', "so-eval", "so-helix", "so-heavynode", "so-standalone", "so-import"] %} - type: udp enabled: true host: "0.0.0.0:514" @@ -253,7 +253,7 @@ output.{{ type }}: {%- endfor %} {%- else %} #----------------------------- Elasticsearch/Logstash output --------------------------------- - {%- if grains['role'] == "so-eval" %} + {%- if grains['role'] in ["so-eval", "so-import"] %} output.elasticsearch: enabled: true hosts: ["{{ MANAGER }}:9200"] diff --git a/setup/so-setup b/setup/so-setup index 4a8cf259e..9c28ab814 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -279,7 +279,7 @@ if [[ $is_helix || $is_manager || $is_import ]]; then whiptail_homenet_manager fi -if [[ $is_helix || $is_manager || $is_node ]]; then +if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then set_base_heapsizes fi From dfd3a1de6aedebdd2cf77c063283bb3c8207c4aa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 10:42:07 -0400 Subject: [PATCH 28/38] set monitor interface to bond0 for import node --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index 9c28ab814..d64a86549 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -251,6 +251,7 @@ if [[ $is_import ]]; then HNSENSOR=inherit MANAGERUPDATES=0 MANAGERADV=BASIC + INTERFACE=bond0 ZEEKVERSION=ZEEK NIDS=Suricata RULESETUP=ETOPEN From b5c9d44d91f352609f2eeeeda6b14776ae8532ba Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 11:15:14 -0400 Subject: [PATCH 29/38] nginx config for import node --- .../etc/{nginx.conf.so-importpcap => nginx.conf.so-import} | 0 setup/so-functions | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename salt/nginx/etc/{nginx.conf.so-importpcap => nginx.conf.so-import} (100%) diff --git a/salt/nginx/etc/nginx.conf.so-importpcap b/salt/nginx/etc/nginx.conf.so-import similarity index 100% rename from salt/nginx/etc/nginx.conf.so-importpcap rename to salt/nginx/etc/nginx.conf.so-import diff --git a/setup/so-functions b/setup/so-functions index 26d80c98a..35617d492 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1775,7 +1775,7 @@ es_heapsize() { fi export ES_HEAP_SIZE - if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE)$ ]]; then + if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE|IMPORT)$ ]]; then NODE_ES_HEAP_SIZE=ES_HEAP_SIZE export NODE_ES_HEAP_SIZE fi From 41afe0ab2e1c94821ccc80ed776fc119c8b2dfa3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 11:33:10 -0400 Subject: [PATCH 30/38] remove tab --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 35617d492..d3ddc3daa 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -721,7 +721,7 @@ docker_seed_registry() { if [ "$install_type" == 'IMPORT' ]; then local TRUSTED_CONTAINERS=(\ "so-logstash:$VERSION" \ - "so-idstools:$VERSION" \ + "so-idstools:$VERSION" \ "so-nginx:$VERSION" \ "so-filebeat:$VERSION" \ "so-suricata:$VERSION" \ From c166bc84f3e945abe424ab3fe9692e34e0275f1b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 11:48:22 -0400 Subject: [PATCH 31/38] add zeek to import node top --- salt/top.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index a51978b1e..01eed5343 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -401,7 +401,8 @@ base: - suricata.manager - elasticsearch - kibana - - suricata - filebeat - utility + - suricata + - zeek - schedule From dcd5e95b38917c20ba80752965cd526b3519bef5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 11:57:13 -0400 Subject: [PATCH 32/38] add so-pcaptools to registry for import node --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d3ddc3daa..61fa60521 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -730,7 +730,8 @@ docker_seed_registry() { "so-kibana:$VERSION" \ "so-kratos:$VERSION" \ "so-suricata:$VERSION" \ - "so-registry:$VERSION" \ + "so-registry:$VERSION" \ + "so-pcaptools:$VERSION" \ "so-zeek:$VERSION" ) else From 68f5c1c3c54829392a956b0ffca65d89cd1c5ee3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 12:01:25 -0400 Subject: [PATCH 33/38] create web user during setup for import node --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index d64a86549..5b2caed27 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -662,7 +662,7 @@ fi filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 - if [[ $is_manager ]]; then + if [[ $is_manager || $is_import ]]; then set_progress_str 87 'Adding user to SOC' add_web_user >> $setup_log 2>&1 fi From ddf3e6f943eff256c4115fa65b7ea9427cc2b0a7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 14:05:28 -0400 Subject: [PATCH 34/38] remove logstash from docker registry seed --- setup/so-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 61fa60521..ddf771b7c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -720,7 +720,6 @@ docker_seed_registry() { if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then if [ "$install_type" == 'IMPORT' ]; then local TRUSTED_CONTAINERS=(\ - "so-logstash:$VERSION" \ "so-idstools:$VERSION" \ "so-nginx:$VERSION" \ "so-filebeat:$VERSION" \ From 5d5fcecdca8eff6ec99f97cd94d3f131213cf8c0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 15:46:34 -0400 Subject: [PATCH 35/38] set the cluster for import node --- salt/utility/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utility/init.sls b/salt/utility/init.sls index 00899f69a..4779f9c1d 100644 --- a/salt/utility/init.sls +++ b/salt/utility/init.sls @@ -10,7 +10,7 @@ crossclusterson: - template: jinja {% endif %} -{% if grains['role'] == 'so-eval' %} +{% if grains['role'] in ['so-eval', 'so-import'] %} fixsearch: cmd.script: - shell: /bin/bash From f59b8683ae87b42f5278a53cbca537f3a139180f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 15:48:34 -0400 Subject: [PATCH 36/38] allow soup to run on import node --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 608394530..0414ceb6e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -27,7 +27,7 @@ exec 3>&1 1>${SOUP_LOG} 2>&1 manager_check() { # Check to see if this is a manager MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') - if [[ "$MANAGERCHECK" =~ ^('so-eval'|'so-manager'|'so-standalone'|'so-managersearch')$ ]]; then + if [[ "$MANAGERCHECK" =~ ^('so-eval'|'so-manager'|'so-standalone'|'so-managersearch'|'so-import')$ ]]; then echo "This is a manager. We can proceed." else echo "Please run soup on the manager. The manager controls all updates." From a746d597bb0b3bdda772244fd49bafe5eea4d69b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Aug 2020 17:42:45 -0400 Subject: [PATCH 37/38] rename to .yml --- salt/elasticsearch/files/{sotls.yaml => sotls.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/elasticsearch/files/{sotls.yaml => sotls.yml} (100%) diff --git a/salt/elasticsearch/files/sotls.yaml b/salt/elasticsearch/files/sotls.yml similarity index 100% rename from salt/elasticsearch/files/sotls.yaml rename to salt/elasticsearch/files/sotls.yml From 9fafd5f72113c58e4eb47c33a95f10fb09b7d314 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 08:32:51 -0400 Subject: [PATCH 38/38] update trusted containers for soup to minimize downloaded containers --- salt/common/tools/sbin/soup | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0414ceb6e..8f0325a6c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -93,7 +93,21 @@ pillar_changes() { update_dockers() { # List all the containers - if [ $MANAGERCHECK != 'so-helix' ]; then + if [ $MANAGERCHECK == 'so-import' ]; then + TRUSTED_CONTAINERS=( \ + "so-idstools" \ + "so-nginx" \ + "so-filebeat" \ + "so-suricata" \ + "so-soc" \ + "so-elasticsearch" \ + "so-kibana" \ + "so-kratos" \ + "so-suricata" \ + "so-registry" \ + "so-pcaptools" \ + "so-zeek" ) + elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( \ "so-acng" \ "so-thehive-cortex" \