From 946cf81a27d6883b07a6a42cc55a5d24536791fe Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Oct 2021 09:47:25 -0400 Subject: [PATCH 001/110] If ANALYST is selected immediately quit setup --- setup/so-setup | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 7d5ae0a0e..998bdbe05 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -256,7 +256,8 @@ elif [ "$install_type" = 'HELIXSENSOR' ]; then elif [ "$install_type" = 'IMPORT' ]; then is_import=true elif [ "$install_type" = 'ANALYST' ]; then - is_analyst=true + cd .. || exit 255 + exec bash so-analyst-install fi if [[ $is_manager || $is_import ]]; then @@ -337,11 +338,6 @@ if ! [[ -f $install_opt_file ]]; then exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi - if [[ $is_analyst ]]; then - cd .. || exit 255 - exec bash so-analyst-install - fi - else rm -rf $install_opt_file >> "$setup_log" 2>&1 fi From af687b070640ff87c6f50f1ee9ae8ff652832f3c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Oct 2021 10:10:34 -0400 Subject: [PATCH 002/110] Remove all holds on Ubuntu reinstall --- setup/so-functions | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index f7d489f42..c808cc6b2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2037,7 +2037,7 @@ reinstall_init() { if command -v docker &> /dev/null; then # Stop and remove all so-* containers so files can be changed with more safety - if [ $(docker ps -a -q --filter "name=so-" | wc -l) -gt 0 ]; then + if [[ $(docker ps -a -q --filter "name=so-" | wc -l) -gt 0 ]]; then docker stop $(docker ps -a -q --filter "name=so-") docker rm -f $(docker ps -a -q --filter "name=so-") fi @@ -2056,6 +2056,10 @@ reinstall_init() { # Remove the old launcher package in case the config changes remove_package launcher-final + if [[ $OS == 'ubuntu' ]]; then + apt-mark unhold $(apt-mark showhold) + fi + } >> "$setup_log" 2>&1 } From 36e17952950156ce876b4483d2eedd5409434bb8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Oct 2021 10:19:47 -0400 Subject: [PATCH 003/110] Add end of setup log messages per #5032 --- setup/so-functions | 4 ++++ setup/so-setup | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c808cc6b2..30ccc2761 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1537,6 +1537,10 @@ install_cleanup() { info "Removing so-setup permission entry from sudoers file" sed -i '/so-setup/d' /etc/sudoers fi + + if [[ -z $SO_ERROR ]]; then + echo "Setup completed at $(date)" >> "$setup_log" 2>&1 + fi } import_registry_docker() { diff --git a/setup/so-setup b/setup/so-setup index 998bdbe05..d050b80ff 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -935,7 +935,6 @@ if [[ -n $SO_ERROR ]]; then SKIP_REBOOT=1 whiptail_setup_failed - else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { @@ -965,9 +964,9 @@ else } | whiptail_gauge_post_setup "Running post-installation steps..." + echo "Post-installation steps have completed. Awaiting user input to clean up installer." >> $setup_log 2>&1 whiptail_setup_complete [[ $setup_type != 'iso' ]] && whitpail_ssh_warning - echo "Post-installation steps have completed." >> $setup_log 2>&1 fi install_cleanup >> "$setup_log" 2>&1 From b97361fab976d2d7076958b878d06652e80f18e0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Oct 2021 10:23:39 -0400 Subject: [PATCH 004/110] Remove references to xenial in setup Resolves #4292 --- setup/so-functions | 31 +++++++++---------------------- setup/so-preflight | 3 +-- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 30ccc2761..32e84bd9d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1207,11 +1207,7 @@ docker_install() { retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 ;; esac - if [ $OSVER != "xenial" ]; then - retry 50 10 "apt-get -y install docker-ce python3-docker" >> "$setup_log" 2>&1 || exit 1 - else - retry 50 10 "apt-get -y install docker-ce python-docker" >> "$setup_log" 2>&1 || exit 1 - fi + retry 50 10 "apt-get -y install docker-ce python3-docker" >> "$setup_log" 2>&1 || exit 1 fi docker_registry { @@ -2172,10 +2168,10 @@ saltify() { DEBIAN_FRONTEND=noninteractive retry 50 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1 if [ $OSVER == "bionic" ]; then - # Switch to Python 3 as default if this is not xenial + # Switch to Python 3 as default for bionic update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1 elif [ $OSVER == "focal" ]; then - # Switch to Python 3 as default if this is not xenial + # Switch to Python 3 as default for focal update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 >> "$setup_log" 2>&1 fi @@ -2194,21 +2190,16 @@ saltify() { # Grab the version from the os-release file local ubuntu_version ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}') - if [ "$OSVER" != "xenial" ]; then local py_ver_url_path="/py3"; else local py_ver_url_path="/apt"; fi case "$install_type" in 'FLEET') - if [[ $OSVER != 'xenial' ]]; then - retry 50 10 "apt-get -y install python3-mysqldb" >> "$setup_log" 2>&1 || exit 1 - else - retry 50 10 "apt-get -y install python-mysqldb" >> "$setup_log" 2>&1 || exit 1 - fi + retry 50 10 "apt-get -y install python3-mysqldb" >> "$setup_log" 2>&1 || exit 1 ;; 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR') # Add saltstack repo(s) - wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 - echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" + wget -q --inet4-only -O - https://repo.saltstack.com/py3/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 + echo "deb http://repo.saltstack.com/py3/ubuntu/$ubuntu_version/amd64/archive/3003 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" # Add Docker repo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - >> "$setup_log" 2>&1 @@ -2216,7 +2207,7 @@ saltify() { # Get gpg keys mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 - wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com$py_ver_url_path/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 + wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg >> "$setup_log" 2>&1 wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH >> "$setup_log" 2>&1 @@ -2240,7 +2231,7 @@ saltify() { echo "Using apt-key add to add SALTSTACK-GPG-KEY.pub and GPG-KEY-WAZUH" >> "$setup_log" 2>&1 apt-key add "$temp_install_dir"/gpg/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1 apt-key add "$temp_install_dir"/gpg/GPG-KEY-WAZUH >> "$setup_log" 2>&1 - echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" + echo "deb http://repo.saltstack.com/py3/ubuntu/$ubuntu_version/amd64/archive/3003/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" ;; esac @@ -2249,11 +2240,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 - if [[ $OSVER != 'xenial' ]]; then - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb" >> "$setup_log" 2>&1 || exit 1 - else - retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-packaging python-influxdb" >> "$setup_log" 2>&1 || exit 1 - fi + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb" >> "$setup_log" 2>&1 || exit 1 fi } diff --git a/setup/so-preflight b/setup/so-preflight index e58feebd2..2943191eb 100644 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -54,11 +54,10 @@ check_new_repos() { else local ubuntu_version ubuntu_version=$(grep VERSION_ID /etc/os-release 2> /dev/null | awk -F '[ "]' '{print $2}') - if [ "$OSVER" != "xenial" ]; then local py_ver_url_path="/py3"; else local py_ver_url_path="/apt"; fi local repo_arr=( "https://download.docker.com/linux/ubuntu/gpg" "https://download.docker.com/linux/ubuntu" - "https://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com/py3/ubuntu/$ubuntu_version/amd64/archive/3003/SALTSTACK-GPG-KEY.pub" "https://packages.wazuh.com/key/GPG-KEY-WAZUH" "https://packages.wazuh.com" ) From e7581036f7166836574dc973ade4f7c875007077 Mon Sep 17 00:00:00 2001 From: Pete Date: Wed, 13 Oct 2021 13:51:05 -0400 Subject: [PATCH 005/110] remove tty/interactive flags This call to docker exec simply returns a number. No interaction (stdin) or tty is required. Specifically, having the -t option prevents running via salt using a command such as: > salt '*' cmd.run 'so-redis-count' --- salt/common/tools/sbin/so-redis-count | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-redis-count b/salt/common/tools/sbin/so-redis-count index 6ae52f1f6..afcb07d55 100755 --- a/salt/common/tools/sbin/so-redis-count +++ b/salt/common/tools/sbin/so-redis-count @@ -17,4 +17,4 @@ . /usr/sbin/so-common -docker exec -it so-redis redis-cli llen logstash:unparsed +docker exec so-redis redis-cli llen logstash:unparsed From a9b250c0f44cfed6572d9333ae90cbb9dd0960c8 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 13 Oct 2021 21:37:59 +0000 Subject: [PATCH 006/110] Add EG firewall config --- files/firewall/hostgroups.local.yaml | 4 ++++ salt/firewall/assigned_hostgroups.map.yaml | 11 ++++++++++- salt/firewall/portgroups.yaml | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/files/firewall/hostgroups.local.yaml b/files/firewall/hostgroups.local.yaml index 794105627..d02d7c785 100644 --- a/files/firewall/hostgroups.local.yaml +++ b/files/firewall/hostgroups.local.yaml @@ -16,6 +16,10 @@ firewall: ips: delete: insert: + endgame: + ips: + delete: + insert: fleet: ips: delete: diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index d3fb79117..d5fca081f 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -162,6 +162,9 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + endgame: + portgroups: + - {{ portgroups.endgame }} osquery_endpoint: portgroups: - {{ portgroups.fleet_api }} @@ -248,6 +251,9 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + endgame: + portgroups: + - {{ portgroups.endgame }} osquery_endpoint: portgroups: - {{ portgroups.fleet_api }} @@ -337,6 +343,9 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + endgame: + portgroups: + - {{ portgroups.endgame }} osquery_endpoint: portgroups: - {{ portgroups.fleet_api }} @@ -594,4 +603,4 @@ role: - {{ portgroups.all }} minion: portgroups: - - {{ portgroups.salt_manager }} \ No newline at end of file + - {{ portgroups.salt_manager }} diff --git a/salt/firewall/portgroups.yaml b/salt/firewall/portgroups.yaml index 1386267f5..d81d00d6a 100644 --- a/salt/firewall/portgroups.yaml +++ b/salt/firewall/portgroups.yaml @@ -39,6 +39,9 @@ firewall: elasticsearch_rest: tcp: - 9200 + endgame: + tcp: + - 3765 fleet_api: tcp: - 8090 From 78e88e076565efc95d72353b6fa68ce74b9a66b7 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 13 Oct 2021 21:42:54 +0000 Subject: [PATCH 007/110] Add EG firewall allowance via setup --- setup/so-setup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/so-setup b/setup/so-setup index 7d5ae0a0e..71401602e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -967,6 +967,11 @@ else so-learn enable logscan --apply >> $setup_log 2>&1 fi + if [[ -n $ENDGAME_SMP_IP ]]; then + set_progress_str 99 'Configuring firewall for Endgame SMP' + so-firewall --apply includehost endgame $ENDGAME_SMP_IP >> $setup_log 2>&1 + fi + } | whiptail_gauge_post_setup "Running post-installation steps..." whiptail_setup_complete From 5ee0ea3fe72c6ffc7c9e082e9f28893f8de083a3 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 14 Oct 2021 13:59:55 +0000 Subject: [PATCH 008/110] Allow SOC actions to use Jinja --- salt/soc/init.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 69cc54c82..1fbfe5de2 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -26,6 +26,15 @@ soclogdir: - group: 939 - makedirs: True +socactions: + file.managed: + - name: /opt/so/conf/soc/menu.actions.json + - source: salt://soc/files/soc/menu.actions.json + - user: 939 + - group: 939 + - mode: 600 + - template: jinja + socconfig: file.managed: - name: /opt/so/conf/soc/soc.json From 15049f44b993239f29fb8fa0f79b5d2c935c475d Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 14 Oct 2021 15:15:23 +0000 Subject: [PATCH 009/110] Add EG pivot --- salt/soc/files/soc/menu.actions.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/menu.actions.json b/salt/soc/files/soc/menu.actions.json index 665ca4c39..0c144c3cb 100644 --- a/salt/soc/files/soc/menu.actions.json +++ b/salt/soc/files/soc/menu.actions.json @@ -1,3 +1,4 @@ +{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} [ { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", "links": [ @@ -29,5 +30,12 @@ { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", "links": [ "https://www.virustotal.com/gui/search/{value}" - ]} + ]} + {%- if HIGHLANDER %} + {%- set EGHOST = salt['pillar.get']('soc:endgamehost', 'EGHOSTNOTPOPULATED') %} + ,{ "name": "Endgame", "description": "Endgame Endpoint Investigation and Response", "icon": "fa-external-link-alt", "target": "_blank", + "links": [ + "https://{{ EGHOST }}/endpoints/{:agent.id}" + ]} + {% endif %} ] From 6cdc214582feca6ed6d7632526253a8a44865a41 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 14 Oct 2021 15:33:37 +0000 Subject: [PATCH 010/110] Add pillar in setup and change name of EG variable --- setup/so-functions | 7 +++++++ setup/so-setup | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4398cfbcc..3a1d05757 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1640,6 +1640,13 @@ manager_pillar() { printf '%s\n'\ " kratoskey: '$KRATOSKEY'"\ "" >> "$pillar_file" + + if [[ -n $ENDGAMEHOST ]]; then + printf '%s\n'\ + "soc:"\ + " endgamehost: '$ENDGAMEHOST'"\ + "" >> "$pillar_file" + fi } manager_global() { diff --git a/setup/so-setup b/setup/so-setup index 71401602e..95cf84b27 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -967,9 +967,9 @@ else so-learn enable logscan --apply >> $setup_log 2>&1 fi - if [[ -n $ENDGAME_SMP_IP ]]; then + if [[ -n $ENDGAMEHOST ]]; then set_progress_str 99 'Configuring firewall for Endgame SMP' - so-firewall --apply includehost endgame $ENDGAME_SMP_IP >> $setup_log 2>&1 + so-firewall --apply includehost endgame $ENDGAMEHOST >> $setup_log 2>&1 fi } | whiptail_gauge_post_setup "Running post-installation steps..." From f1fafa015edc8c0b52897546256ba3846a2b628d Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 14 Oct 2021 16:27:28 +0000 Subject: [PATCH 011/110] Add EG to list of groups to include 127.0.0.1 --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 3a1d05757..27c4daf95 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1421,7 +1421,7 @@ firewall_generate_templates() { cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1 - for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do + for i in analyst beats_endpoint endgame sensor manager minion osquery_endpoint search_node wazuh_endpoint; do $default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1 done From cba97802fe9c1ff95bb3ae9eb355d05fd9094724 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Oct 2021 15:13:34 -0400 Subject: [PATCH 012/110] Fix indent --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index e44429da6..af66896c6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1225,7 +1225,7 @@ docker_registry() { mkdir -p /etc/docker >> "$setup_log" 2>&1 # This will get applied so docker can attempt to start if [ -z "$DOCKERNET" ]; then - DOCKERNET=172.17.0.0 + DOCKERNET=172.17.0.0 fi # Make the host use the manager docker registry DNETBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 From c7b15a9b1ff2b2640c490f65d394de520a129876 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Oct 2021 15:13:55 -0400 Subject: [PATCH 013/110] Replace rather than append to Kibana misc log --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index d42596287..cf130e003 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -14,4 +14,4 @@ wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELAST SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # Load saved objects -{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log +{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /opt/so/log/kibana/misc.log From 9797a152189d3f53e8a83f9952f0aff4fba314e9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 14 Oct 2021 17:23:18 -0400 Subject: [PATCH 014/110] Fix issue with 'so-user delete' resetting all user roles - note that this function is not technically supported or published since it's not intended for production use --- salt/common/tools/sbin/so-user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 015a28c9f..5a52a9d59 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -445,7 +445,7 @@ function deleteUser() { rolesTmpFile="${socRolesFile}.tmp" createFile "$rolesTmpFile" "$soUID" "$soGID" - grep -v "$id" "$socRolesFile" > "$rolesTmpFile" + grep -v "$identityId" "$socRolesFile" > "$rolesTmpFile" mv "$rolesTmpFile" "$socRolesFile" } From 1800ec4570f2045f16d10f88116df5b3985aa4c8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 15 Oct 2021 09:25:44 -0400 Subject: [PATCH 015/110] Upgrade to Elastalert 2 v2.2.2 --- salt/common/tools/sbin/so-elastalert-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastalert-test b/salt/common/tools/sbin/so-elastalert-test index 51d00e6fa..2d3106ab6 100755 --- a/salt/common/tools/sbin/so-elastalert-test +++ b/salt/common/tools/sbin/so-elastalert-test @@ -70,7 +70,7 @@ do done docker_exec(){ - CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/config/elastalert_config.yaml $OPTIONS" + CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/elastalert/config.yaml $OPTIONS" if [ "${RESULTS_TO_LOG,,}" = "y" ] ; then $CMD > "$FILE_SAVE_LOCATION" else From 032373187cc577f721d2b8b95b426da6a98fc00a Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 15 Oct 2021 16:02:53 +0000 Subject: [PATCH 016/110] Allow setting ES index patterns for SOC in pillar --- salt/soc/files/soc/soc.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index e33ea406b..02128fd3c 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -26,7 +26,7 @@ {%- set ES_USER = '' %} {%- set ES_PASS = '' %} {%- endif %} - +{%- set ES_INDEX_PATTERNS = salt['pillar.get']('soc:es_index_patterns', '*:so-*') %} { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", "server": { @@ -57,6 +57,7 @@ {%- endif %} "username": "{{ ES_USER }}", "password": "{{ ES_PASS }}", + "index": "{{ ES_INDEX_PATTERNS }}", "cacheMs": {{ ES_FIELDCAPS_CACHE }}, "verifyCert": false, "timeoutMs": {{ API_TIMEOUT }} From 8feeff97b5869397ad902736c7768087d3474537 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 15 Oct 2021 16:19:19 +0000 Subject: [PATCH 017/110] Add EG index pattern during setup (if enabled) --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index af66896c6..3f6a2b136 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1645,6 +1645,7 @@ manager_pillar() { printf '%s\n'\ "soc:"\ " endgamehost: '$ENDGAMEHOST'"\ + " es_index_patterns: '*:so-*, *:endgame-*'"\ "" >> "$pillar_file" fi } From 8de8d5815516a6e14575e699c8a8e5f366c2699f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 15 Oct 2021 13:27:08 -0400 Subject: [PATCH 018/110] Upgrade to ES 7.15.1 --- salt/kibana/files/saved_objects.ndjson.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/saved_objects.ndjson.jinja b/salt/kibana/files/saved_objects.ndjson.jinja index abc722a9e..9cf037b78 100644 --- a/salt/kibana/files/saved_objects.ndjson.jinja +++ b/salt/kibana/files/saved_objects.ndjson.jinja @@ -462,7 +462,7 @@ {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} {% endraw -%} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.0","id":"7.15.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":{{DASHBOARD.discover.sampleSize}},"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.15.1","id":"7.15.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {% raw -%} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} From 59852841ffe9628b3833002de7d2b67dda8bd8b0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 15 Oct 2021 13:29:50 -0400 Subject: [PATCH 019/110] Add keyword subfield for event.module --- .../templates/so/so-endgame-template.json.jinja | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja index 824558e8f..e39a2fcf9 100644 --- a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja @@ -719,7 +719,12 @@ }, "module": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "original": { "doc_values": false, From a9f6c84d7cf5d251366fb1fc349ac98bab4f329a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 15 Oct 2021 14:17:54 -0400 Subject: [PATCH 020/110] Add Steno Tuning Options --- salt/pcap/files/config | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/salt/pcap/files/config b/salt/pcap/files/config index 900234bc1..24f9a579e 100644 --- a/salt/pcap/files/config +++ b/salt/pcap/files/config @@ -1,20 +1,23 @@ -{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} -{%- set diskfreepercentage = salt['pillar.get']('steno:diskfreepercentage', 10) %} -{%- set maxfiles = salt['pillar.get']('steno:maxfiles', 30000) %} - - +{%- set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %} +{%- set DISKFREEPERCENTAGE = salt['pillar.get']('steno:diskfreepercentage', 10) %} +{%- set MAXFILES = salt['pillar.get']('steno:maxfiles', 30000) %} +{%- set BLOCKS = salt['pillar.get']('steno:blocks', 2048) %} +{%- set FILEMB = salt['pillar.get']('steno:filemb', 4096) %} +{%- set AIOPS = salt['pillar.get']('steno:aiops', 128) %} +{%- set THREADS = salt['pillar.get']('steno:threads', 1) %} { "Threads": [ - { "PacketsDirectory": "/nsm/pcap" - , "IndexDirectory": "/nsm/pcapindex" - , "MaxDirectoryFiles": {{ maxfiles }} - , "DiskFreePercentage": {{ diskfreepercentage }} - } + { "PacketsDirectory": "/nsm/pcap", "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} } + {%- if THREADS > 1 %} + {%- for i in range(2,THREADS+1) %} + , { "PacketsDirectory": "/nsm/pcap" , "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ MAXFILES }}, "DiskFreePercentage": {{ DISKFREEPERCENTAGE }} } + {%- endfor %} + {%- endif %} ] , "StenotypePath": "/usr/bin/stenotype" - , "Interface": "{{ interface }}" + , "Interface": "{{ INTERFACE }}" , "Port": 1234 , "Host": "127.0.0.1" - , "Flags": ["-v", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] + , "Flags": ["-v", "--blocks={{ BLOCKS }}", "--preallocate_file_mb={{ FILEMB }}", "--aiops={{ AIOPS }}", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] , "CertPath": "/etc/stenographer/certs" } From d0a6dafc8bc49d22797c86cb060317eaf11bb278 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 15 Oct 2021 17:09:59 -0400 Subject: [PATCH 021/110] Add TI module --- salt/common/tools/sbin/so-filebeat-module-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 401f54289..14a3ebed2 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -54,7 +54,7 @@ PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_ if [[ "$PIPELINES" -lt 5 ]]; then echo "Setting up ingest pipeline(s)" - for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler + for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler do echo "Loading $MODULE" docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML From bb36fc1ed8e594378b8d1758209f7895afa56f05 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 15 Oct 2021 17:16:38 -0400 Subject: [PATCH 022/110] Add TI module defaults --- salt/filebeat/thirdpartydefaults.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index 112ed6d6c..3be8bb901 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -244,6 +244,23 @@ third_party_filebeat: var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9501 + threatintel: + abuseurl: + enabled: false + abusemalware: + enabled: false + misp: + enabled: false + malwarebazaar: + enabled: false + otx: + enabled: false + anomali: + enabled: false + anomalithreatstream: + enabled: false + recordedfuture: + enabled: false zscaler: zia: enabled: false From b9a3d3a6a96f415498f1c8b1ca0918d4d74a30d6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 19 Oct 2021 11:14:02 -0400 Subject: [PATCH 023/110] Fix Steno Math for PL --- salt/telegraf/scripts/stenoloss.sh | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index 028637e16..cd09b952f 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -19,25 +19,30 @@ THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then - TSFILE=/var/log/telegraf/laststenodrop.log - if [ -f "$TSFILE" ]; then - LASTTS=$(cat $TSFILE) - else - LASTTS=0 + CHECKIT=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2|wc -l) + STENOGREP=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2) + + declare RESULT=($STENOGREP) + + CURRENT_PACKETS=$(echo ${RESULT[9]} | awk -F'=' '{print $2 }') + CURRENT_DROPS=$(echo ${RESULT[12]} | awk -F'=' '{print $2 }') + PREVIOUS_PACKETS=$(echo ${RESULT[23]} | awk -F'=' '{print $2 }') + PREVIOUS_DROPS=$(echo ${RESULT[26]} | awk -F'=' '{print $2 }') + + DROPPED=$((CURRENT_DROPS - PREVIOUS_DROPS)) + TOTAL_CURRENT=$((CURRENT_PACKETS + CURRENT_DROPS)) + TOTAL_PAST=$((PREVIOUS_PACKETS + PREVIOUS_DROPS)) + TOTAL=$((TOTAL_CURRENT - TOTAL_PAST)) + + if [ $CHECKIT == 2 ]; then + if [ $DROPPED == 0 ]; then + echo "stenodrop drop=$DROPPED" + else + LOSS=$(echo "4 k $DROPPED $TOTAL / 100 * p" | dc) + echo "stenodrop drop=$loss" + fi fi - - # Get the data - LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) - CURRENTTS=$(echo $LOGLINE | awk '{print $1}') - - if [[ "$CURRENTTS" != "$LASTTS" ]]; then - DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') - echo $CURRENTTS > $TSFILE - else - DROP=0 - fi - - echo "stenodrop drop=$DROP" + else exit 0 fi \ No newline at end of file From 64f25961b0b785b80031e53b7da54af0c2ac3b59 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 19 Oct 2021 11:15:58 -0400 Subject: [PATCH 024/110] Fix Steno Math for PL --- salt/telegraf/scripts/stenoloss.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index cd09b952f..09b7ddb70 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -39,7 +39,7 @@ if [ ! "$THEGREP" ]; then echo "stenodrop drop=$DROPPED" else LOSS=$(echo "4 k $DROPPED $TOTAL / 100 * p" | dc) - echo "stenodrop drop=$loss" + echo "stenodrop drop=$LOSS" fi fi From 9453ed7fa11fbf0c5908105d987615a8fbab0240 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 19 Oct 2021 13:01:40 -0400 Subject: [PATCH 025/110] Remove space to allow pattern(s) to be correctly interpreted --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 3f6a2b136..f4d08e9a9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1645,7 +1645,7 @@ manager_pillar() { printf '%s\n'\ "soc:"\ " endgamehost: '$ENDGAMEHOST'"\ - " es_index_patterns: '*:so-*, *:endgame-*'"\ + " es_index_patterns: '*:so-*,*:endgame-*'"\ "" >> "$pillar_file" fi } From 2f8bb5a2a6729c13d9e2233dfe81821dd8a1a580 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 19 Oct 2021 16:04:10 -0400 Subject: [PATCH 026/110] Fix Docker-created corruption of SOC user roles file --- salt/common/tools/sbin/so-user | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 5a52a9d59..9bf36cf99 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -182,6 +182,10 @@ function ensureRoleFileExists() { echo "Database file does not exist yet, installation is likely not yet complete." fi + if [[ -d "$socRolesFile" ]]; then + echo "Removing invalid roles directory created by Docker" + rm -fr "$socRolesFile" + fi mv "${rolesTmpFile}" "${socRolesFile}" fi } From b496810b63fc6d0d8bc3dba0092b402554a6ec92 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 14:46:47 -0400 Subject: [PATCH 027/110] add redis and logstash input plugins to telegraf --- salt/telegraf/etc/telegraf.conf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index b04b1549d..71fc610b4 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -639,6 +639,21 @@ insecure_skip_verify = true {% endif %} +{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.logstash]] + url = "http://localhost:9600" + collect = ["pipelines"] + {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + username = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:pass') }}" + {% endif %} +{%- endif %} + +{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.redis]] + servers = "tcp://localhost:5000" +{%- endif %} + # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" From 7ecfb55b7026e6858512de04b25c6c137c4b42c4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 14:50:50 -0400 Subject: [PATCH 028/110] fix pillar call --- salt/telegraf/etc/telegraf.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 71fc610b4..dac41123e 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -644,8 +644,8 @@ url = "http://localhost:9600" collect = ["pipelines"] {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} - username = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:user') }}" - password = "{{ salt['pillar.get']('elasticsearch:auth:so_logstash_user:pass') }}" + username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" {% endif %} {%- endif %} From 8619af59ccee3a5ed24aede22a37bcf1de1e0c55 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 15:02:33 -0400 Subject: [PATCH 029/110] servers to list format --- salt/telegraf/etc/telegraf.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index dac41123e..b92ba9f33 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -651,7 +651,7 @@ {% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} [[inputs.redis]] - servers = "tcp://localhost:5000" + servers = ["tcp://localhost:5000"] {%- endif %} # From adffb1180045d483df72dfb989ef05ae6ded41cc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 15:39:21 -0400 Subject: [PATCH 030/110] fix redis port --- salt/telegraf/etc/telegraf.conf | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index b92ba9f33..59d806fe0 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -639,21 +639,6 @@ insecure_skip_verify = true {% endif %} -{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} -[[inputs.logstash]] - url = "http://localhost:9600" - collect = ["pipelines"] - {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} - username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" - password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" - {% endif %} -{%- endif %} - -{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} -[[inputs.redis]] - servers = ["tcp://localhost:5000"] -{%- endif %} - # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" @@ -688,9 +673,22 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false +{% if grains.role in ['so-node','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.logstash]] + url = "http://localhost:9600" + collect = ["pipelines"] + {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" + {% endif %} +{%- endif %} + +{% if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode'] -%} +[[inputs.redis]] + servers = ["tcp://localhost:6379"] +{%- endif %} # # Read metrics from one or more commands that can output to stdout - [[inputs.exec]] commands = [ "/scripts/sostatus.sh" From b5cb47e0663275dac79ceca221c94f4865a687c6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 20 Oct 2021 16:43:55 -0400 Subject: [PATCH 031/110] Fix sbin perms --- salt/common/tools/sbin/so-elasticsearch-roles-load | 0 salt/common/tools/sbin/so-import-evtx | 0 salt/common/tools/sbin/so-playbook-import | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 salt/common/tools/sbin/so-elasticsearch-roles-load mode change 100644 => 100755 salt/common/tools/sbin/so-import-evtx mode change 100644 => 100755 salt/common/tools/sbin/so-playbook-import diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/common/tools/sbin/so-elasticsearch-roles-load old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-playbook-import b/salt/common/tools/sbin/so-playbook-import old mode 100644 new mode 100755 From 0ed2ce0766decd07def16f38b1a2eb68564c8183 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 20 Oct 2021 16:44:09 -0400 Subject: [PATCH 032/110] Fix validation.sh tests --- tests/validation.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/validation.sh b/tests/validation.sh index d16c8bbb9..6ec2a5247 100644 --- a/tests/validation.sh +++ b/tests/validation.sh @@ -1,6 +1,6 @@ #!/bin/bash -. ../salt/common/tools/sbin/so-common +. "$(dirname "$0")"/../salt/common/tools/sbin/so-common script_ret=0 @@ -106,7 +106,7 @@ test_fun 1 valid_dns_list "192.168.9." sleep 0.15s -header "int (default min: 1, default max: 1000)" +header "int (default min: 1, default max: 1000000000)" test_fun 0 valid_int "24" @@ -114,9 +114,9 @@ test_fun 0 valid_int "1" test_fun 0 valid_int "2" "2" -test_fun 0 valid_int "1000" +test_fun 0 valid_int "1000000000" -test_fun 1 valid_int "10001" +test_fun 1 valid_int "1000000001" test_fun 1 valid_int "24" "" "20" From 387d4d6ad56e438b30d0dd9503d7c132d4c824b7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 20 Oct 2021 16:44:57 -0400 Subject: [PATCH 033/110] Add so-deny script + rewrite so-allow to match so-deny --- salt/common/tools/sbin/so-deny | 209 +++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100755 salt/common/tools/sbin/so-deny diff --git a/salt/common/tools/sbin/so-deny b/salt/common/tools/sbin/so-deny new file mode 100755 index 000000000..c36a9b9d6 --- /dev/null +++ b/salt/common/tools/sbin/so-deny @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ipaddress +import textwrap +import os +import subprocess +import sys +import argparse +import re +from lxml import etree as ET +from xml.dom import minidom + + +LOCAL_SALT_DIR='/opt/so/saltstack/local' +WAZUH_CONF='/nsm/wazuh/etc/ossec.conf' +VALID_ROLES = { + 'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' }, + 'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' }, + 'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' }, + 'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' }, + 'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' }, + 's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' }, + 'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' }, + 'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' }, + 'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' } +} + + +def validate_ip_cidr(ip_cidr: str) -> bool: + try: + ipaddress.ip_address(ip_cidr) + except ValueError: + try: + ipaddress.ip_network(ip_cidr) + except ValueError: + return False + return True + + +def role_prompt() -> str: + print() + print('Choose the role for the IP or Range you would like to deny') + print() + for role in VALID_ROLES: + print(f'[{role}] - {VALID_ROLES[role]["desc"]}') + print() + role = input('Please enter your selection: ') + if role in VALID_ROLES.keys(): + return VALID_ROLES[role]['role'] + else: + print(f'Invalid role \'{role}\', please try again.', file=sys.stderr) + sys.exit(1) + + +def ip_prompt() -> str: + ip = input('Enter a single ip address or range to deny (ex: 10.10.10.10 or 10.10.0.0/16): ') + if validate_ip_cidr(ip): + return ip + else: + print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr) + sys.exit(1) + + +def wazuh_enabled() -> bool: + for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'): + with open(file, 'r') as pillar: + if 'wazuh: 1' in pillar.read(): + return True + return False + + +def root_to_str(root: ET.ElementTree) -> str: + xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '') + xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str) + + # Remove specific substrings to better format comments on intial parse/write + xml_str = re.sub(r' -', '', xml_str) + xml_str = re.sub(r' -->', ' -->', xml_str) + + dom = minidom.parseString(xml_str) + return dom.toprettyxml(indent=" ") + + +def rem_wl(ip): + parser = ET.XMLParser(remove_blank_text=True) + with open(WAZUH_CONF, 'rb') as wazuh_conf: + tree = ET.parse(wazuh_conf, parser) + root = tree.getroot() + + global_elems = root.findall(f"global/white_list[. = '{ip}']/..") + if len(global_elems) > 0: + for g_elem in global_elems: + ge_index = list(root).index(g_elem) + if ge_index > 0 and root[list(root).index(g_elem) - 1].tag == ET.Comment: + root.remove(root[ge_index - 1]) + root.remove(g_elem) + + with open(WAZUH_CONF, 'w') as out: + out.write(root_to_str(root)) + + +def apply(role: str, ip: str) -> int: + firewall_cmd = ['so-firewall', 'excludehost', role, ip] + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True'] + restart_wazuh_cmd = ['so-wazuh-restart'] + print(f'Removing {ip} from the {role} role. This can take a few seconds...') + cmd = subprocess.run(firewall_cmd) + if cmd.returncode == 0: + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + else: + return cmd.returncode + if cmd.returncode == 0: + if wazuh_enabled and role=='analyst': + try: + rem_wl(ip) + print(f'Removed whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + except Exception as e: + print(f'Failed to remove whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + print(e) + return 1 + print('Restarting OSSEC Server...') + cmd = subprocess.run(restart_wazuh_cmd) + else: + return cmd.returncode + else: + print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr) + return cmd.returncode + if cmd.returncode != 0: + print('Failed to restart OSSEC server.') + return cmd.returncode + + +def main(): + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(f'''\ + additional information: + To use this script in interactive mode call it with no arguments + ''' + )) + + group = main_parser.add_argument_group(title='roles') + group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp") + group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp") + group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp") + group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp") + group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp") + group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp") + group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp") + group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp") + group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp") + + ip_g = main_parser.add_argument_group(title='allow') + ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip') + + args = main_parser.parse_args(sys.argv[1:]) + + if args.roles is None: + role = role_prompt() + ip = ip_prompt() + try: + return_code = apply(role, ip) + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + sys.exit(return_code) + elif args.roles is not None and args.ip is None: + main_parser.print_help() + else: + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + + sys.exit(return_code) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.exit(1) From 928aed27c5ab7c4d8a2bd17d563c57269c067f2d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 20 Oct 2021 17:04:02 -0400 Subject: [PATCH 034/110] require files before starting soc or kratos --- salt/soc/init.sls | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 1fbfe5de2..46449b15d 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -80,6 +80,10 @@ soccustomroles: - mode: 600 - template: jinja +socusersroles: + file.exists: + - name: /opt/so/conf/soc/soc_users_roles + # we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' # is in the minion config. That line is added before the final highstate during setup sosyncusers: @@ -95,13 +99,13 @@ so-soc: - name: so-soc - binds: - /nsm/soc/jobs:/opt/sensoroni/jobs:rw + - /opt/so/log/soc/:/opt/sensoroni/logs/:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} - extra_hosts: {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} @@ -112,6 +116,15 @@ so-soc: - 0.0.0.0:9822:9822 - watch: - file: /opt/so/conf/soc/* + - require: + - file: socdatadir + - file: soclogdir + - file: socconfig + - file: socmotd + - file: socbanner + - file: soccustom + - file: soccustomroles + - file: socusersroles append_so-soc_so-status.conf: file.append: @@ -154,6 +167,14 @@ kratossync: - file_mode: 600 - template: jinja +kratos_schema: + file.exists: + - name: /opt/so/conf/kratos/schema.json + +kratos_yaml: + file.exists: + - name: /opt/so/conf/kratos/kratos.yaml + so-kratos: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kratos:{{ VERSION }} @@ -169,6 +190,11 @@ so-kratos: - 0.0.0.0:4434:4434 - watch: - file: /opt/so/conf/kratos + - require: + - file: kratos_schema + - file: kratos_yaml + - file: kratoslogdir + - file: kratosdir append_so-kratos_so-status.conf: file.append: From 0beeeb94bfde67023f3c7310b33b8d6e5bfb435a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 21 Oct 2021 10:48:17 -0400 Subject: [PATCH 035/110] Actually add new so-allow script --- salt/common/tools/sbin/so-allow | 323 ++++++++++++++++++-------------- 1 file changed, 183 insertions(+), 140 deletions(-) diff --git a/salt/common/tools/sbin/so-allow b/salt/common/tools/sbin/so-allow index c3cdc0ea2..1d240d840 100755 --- a/salt/common/tools/sbin/so-allow +++ b/salt/common/tools/sbin/so-allow @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env python3 # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # @@ -15,152 +15,195 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -. /usr/sbin/so-common +import ipaddress +import textwrap +import os +import subprocess +import sys +import argparse +import re +from lxml import etree as ET +from xml.dom import minidom +from datetime import datetime as dt +from datetime import timezone as tz -local_salt_dir=/opt/so/saltstack/local - -SKIP=0 - -function usage { - -cat << EOF - -Usage: $0 [-abefhoprsw] [ -i IP ] - -This program allows you to add a firewall rule to allow connections from a new IP address or CIDR range. - -If you run this program with no arguments, it will present a menu for you to choose your options. - -If you want to automate and skip the menu, you can pass the desired options as command line arguments. - -EXAMPLES - -To add 10.1.2.3 to the analyst role: -so-allow -a -i 10.1.2.3 - -To add 10.1.2.0/24 to the osquery role: -so-allow -o -i 10.1.2.0/24 - -EOF +LOCAL_SALT_DIR='/opt/so/saltstack/local' +WAZUH_CONF='/nsm/wazuh/etc/ossec.conf' +VALID_ROLES = { + 'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' }, + 'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' }, + 'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' }, + 'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' }, + 'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' }, + 's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' }, + 'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' }, + 'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' }, + 'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' } } -while getopts "ahfesprbowi:" OPTION -do - case $OPTION in - h) - usage - exit 0 - ;; - a) - FULLROLE="analyst" - SKIP=1 - ;; - b) - FULLROLE="beats_endpoint" - SKIP=1 - ;; - e) - FULLROLE="elasticsearch_rest" - SKIP=1 - ;; - f) - FULLROLE="strelka_frontend" - SKIP=1 - ;; - i) IP=$OPTARG - ;; - o) - FULLROLE="osquery_endpoint" - SKIP=1 - ;; - w) - FULLROLE="wazuh_agent" - SKIP=1 - ;; - s) - FULLROLE="syslog" - SKIP=1 - ;; - p) - FULLROLE="wazuh_api" - SKIP=1 - ;; - r) - FULLROLE="wazuh_authd" - SKIP=1 - ;; - *) - usage - exit 0 - ;; - esac -done -if [ "$SKIP" -eq 0 ]; then +def validate_ip_cidr(ip_cidr: str) -> bool: + try: + ipaddress.ip_address(ip_cidr) + except ValueError: + try: + ipaddress.ip_network(ip_cidr) + except ValueError: + return False + return True - echo "This program allows you to add a firewall rule to allow connections from a new IP address." - echo "" - echo "Choose the role for the IP or Range you would like to add" - echo "" - echo "[a] - Analyst - ports 80/tcp and 443/tcp" - echo "[b] - Logstash Beat - port 5044/tcp" - echo "[e] - Elasticsearch REST API - port 9200/tcp" - echo "[f] - Strelka frontend - port 57314/tcp" - echo "[o] - Osquery endpoint - port 8090/tcp" - echo "[s] - Syslog device - 514/tcp/udp" - echo "[w] - Wazuh agent - port 1514/tcp/udp" - echo "[p] - Wazuh API - port 55000/tcp" - echo "[r] - Wazuh registration service - 1515/tcp" - echo "" - echo "Please enter your selection:" - read -r ROLE - echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):" - read -r IP - if [ "$ROLE" == "a" ]; then - FULLROLE=analyst - elif [ "$ROLE" == "b" ]; then - FULLROLE=beats_endpoint - elif [ "$ROLE" == "e" ]; then - FULLROLE=elasticsearch_rest - elif [ "$ROLE" == "f" ]; then - FULLROLE=strelka_frontend - elif [ "$ROLE" == "o" ]; then - FULLROLE=osquery_endpoint - elif [ "$ROLE" == "w" ]; then - FULLROLE=wazuh_agent - elif [ "$ROLE" == "s" ]; then - FULLROLE=syslog - elif [ "$ROLE" == "p" ]; then - FULLROLE=wazuh_api - elif [ "$ROLE" == "r" ]; then - FULLROLE=wazuh_authd - else - echo "I don't recognize that role" - exit 1 - fi +def role_prompt() -> str: + print() + print('Choose the role for the IP or Range you would like to allow') + print() + for role in VALID_ROLES: + print(f'[{role}] - {VALID_ROLES[role]["desc"]}') + print() + role = input('Please enter your selection: ') + if role in VALID_ROLES.keys(): + return VALID_ROLES[role]['role'] + else: + print(f'Invalid role \'{role}\', please try again.', file=sys.stderr) + sys.exit(1) + -fi +def ip_prompt() -> str: + ip = input('Enter a single ip address or range to allow (ex: 10.10.10.10 or 10.10.0.0/16): ') + if validate_ip_cidr(ip): + return ip + else: + print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr) + sys.exit(1) -echo "Adding $IP to the $FULLROLE role. This can take a few seconds" -/usr/sbin/so-firewall includehost $FULLROLE $IP -salt-call state.apply firewall queue=True -# Check if Wazuh enabled -if grep -q -R "wazuh: 1" $local_salt_dir/pillar/*; then - # If analyst, add to Wazuh AR whitelist - if [ "$FULLROLE" == "analyst" ]; then - WAZUH_MGR_CFG="/nsm/wazuh/etc/ossec.conf" - if ! grep -q "$IP" $WAZUH_MGR_CFG ; then - DATE=$(date) - sed -i 's/<\/ossec_config>//' $WAZUH_MGR_CFG - sed -i '/^$/N;/^\n$/D' $WAZUH_MGR_CFG - echo -e "\n \n $IP\n \n" >> $WAZUH_MGR_CFG - echo "Added whitelist entry for $IP in $WAZUH_MGR_CFG." - echo - echo "Restarting OSSEC Server..." - /usr/sbin/so-wazuh-restart - fi - fi -fi +def wazuh_enabled() -> bool: + for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'): + with open(file, 'r') as pillar: + if 'wazuh: 1' in pillar.read(): + return True + return False + + +def root_to_str(root: ET.ElementTree) -> str: + xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '') + xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str) + xml_str = re.sub(r' -', '', xml_str) + xml_str = re.sub(r' -->', ' -->', xml_str) + dom = minidom.parseString(xml_str) + return dom.toprettyxml(indent=" ") + + +def add_wl(ip): + parser = ET.XMLParser(remove_blank_text=True) + with open(WAZUH_CONF, 'rb') as wazuh_conf: + tree = ET.parse(wazuh_conf, parser) + root = tree.getroot() + + source_comment = ET.Comment(f'Address {ip} added by /usr/sbin/so-allow on {dt.utcnow().replace(tzinfo=tz.utc).strftime("%a %b %e %H:%M:%S %Z %Y")}') + new_global = ET.Element("global") + new_wl = ET.SubElement(new_global, 'white_list') + new_wl.text = ip + + root.append(source_comment) + root.append(new_global) + + with open(WAZUH_CONF, 'w') as add_out: + add_out.write(root_to_str(root)) + + +def apply(role: str, ip: str) -> int: + firewall_cmd = ['so-firewall', 'includehost', role, ip] + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True'] + restart_wazuh_cmd = ['so-wazuh-restart'] + print(f'Adding {ip} to the {role} role. This can take a few seconds...') + cmd = subprocess.run(firewall_cmd) + if cmd.returncode == 0: + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + else: + return cmd.returncode + if cmd.returncode == 0: + if wazuh_enabled and role=='analyst': + try: + add_wl(ip) + print(f'Added whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + except Exception as e: + print(f'Failed to add whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + print(e) + return 1 + print('Restarting OSSEC Server...') + cmd = subprocess.run(restart_wazuh_cmd) + else: + return cmd.returncode + else: + print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr) + return cmd.returncode + if cmd.returncode != 0: + print('Failed to restart OSSEC server.') + return cmd.returncode + + +def main(): + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(f'''\ + additional information: + To use this script in interactive mode call it with no arguments + ''' + )) + + group = main_parser.add_argument_group(title='roles') + group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp") + group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp") + group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp") + group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp") + group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp") + group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp") + group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp") + group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp") + group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp") + + ip_g = main_parser.add_argument_group(title='allow') + ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip') + + args = main_parser.parse_args(sys.argv[1:]) + + if args.roles is None: + role = role_prompt() + ip = ip_prompt() + try: + return_code = apply(role, ip) + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + sys.exit(return_code) + elif args.roles is not None and args.ip is None: + main_parser.print_help() + else: + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + + sys.exit(return_code) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.exit(1) + From 15fe7512b711c78abda442204b7ab48ff508e49b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 21 Oct 2021 10:49:41 -0400 Subject: [PATCH 036/110] Install lxml during setup and in common state --- salt/common/init.sls | 2 ++ setup/so-functions | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index f993534b3..05dd7023f 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -101,6 +101,7 @@ commonpkgs: - python3-m2crypto - python3-mysqldb - python3-packaging + - python3-lxml - git - vim @@ -143,6 +144,7 @@ commonpkgs: - python36-m2crypto - python36-mysql - python36-packaging + - python36-lxml - yum-utils - device-mapper-persistent-data - lvm2 diff --git a/setup/so-functions b/setup/so-functions index f4d08e9a9..58fbca562 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2164,6 +2164,7 @@ saltify() { python36-m2crypto\ python36-mysql\ python36-packaging\ + python36-lxml\ yum-utils\ device-mapper-persistent-data\ lvm2\ @@ -2250,7 +2251,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" >> "$setup_log" 2>&1 || exit 1 fi } From 77ee1db44ceb5a76f0c7c3487469c18bf86e61e5 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 21 Oct 2021 12:56:03 -0400 Subject: [PATCH 037/110] Add .keyword subfield for conflict fields --- .../so/so-endgame-template.json.jinja | 150 +++++++++++++++--- 1 file changed, 129 insertions(+), 21 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja index e39a2fcf9..6d2b89b27 100644 --- a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja @@ -26,23 +26,48 @@ "properties": { "ephemeral_id": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "id": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "version": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -597,7 +622,12 @@ "properties": { "version": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -683,18 +713,33 @@ }, "category": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "code": { "ignore_above": 1024, "type": "keyword" }, "created": { - "type": "date" + "type": "date", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "dataset": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "duration": { "type": "long" @@ -711,7 +756,12 @@ "type": "keyword" }, "ingested": { - "type": "date" + "type": "date", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "kind": { "ignore_above": 1024, @@ -734,7 +784,12 @@ }, "outcome": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "provider": { "ignore_above": 1024, @@ -761,11 +816,21 @@ }, "timezone": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "url": { "ignore_above": 1024, @@ -1011,7 +1076,12 @@ }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "os": { "properties": { @@ -1144,11 +1214,21 @@ }, "method": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "referrer": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -1192,7 +1272,12 @@ "properties": { "level": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "logger": { "ignore_above": 1024, @@ -2154,7 +2239,12 @@ }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "node": { "properties": { @@ -2170,7 +2260,12 @@ }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "version": { "ignore_above": 1024, @@ -2182,7 +2277,12 @@ "properties": { "address": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "as": { "properties": { @@ -2338,7 +2438,12 @@ }, "tags": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "threat": { "properties": { @@ -2689,6 +2794,9 @@ }, "original": { "fields": { + "keyword": { + "type": "keyword" + }, "text": { "norms": false, "type": "text" From f374dcbb587fdf34a1b2819a225c545235a7614c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 21 Oct 2021 13:54:06 -0400 Subject: [PATCH 038/110] Check for IP environment variable in so-allow and so-deny --- salt/common/tools/sbin/so-allow | 32 ++++++++++++++++++-------------- salt/common/tools/sbin/so-deny | 32 ++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/salt/common/tools/sbin/so-allow b/salt/common/tools/sbin/so-allow index 1d240d840..769dcc1e9 100755 --- a/salt/common/tools/sbin/so-allow +++ b/salt/common/tools/sbin/so-allow @@ -183,21 +183,25 @@ def main(): return_code = e.errno sys.exit(return_code) elif args.roles is not None and args.ip is None: - main_parser.print_help() - else: - if validate_ip_cidr(args.ip): - try: - for role in args.roles: - return_code = apply(role, args.ip) - if return_code > 0: - break - except Exception as e: - print(f'Unexpected exception occurred: {e}', file=sys.stderr) - return_code = e.errno + if os.environ.get('IP') is None: + main_parser.print_help() + sys.exit(1) else: - print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) - return_code = 1 - + args.ip = os.environ['IP'] + + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + sys.exit(return_code) diff --git a/salt/common/tools/sbin/so-deny b/salt/common/tools/sbin/so-deny index c36a9b9d6..c13ea3f32 100755 --- a/salt/common/tools/sbin/so-deny +++ b/salt/common/tools/sbin/so-deny @@ -184,21 +184,25 @@ def main(): return_code = e.errno sys.exit(return_code) elif args.roles is not None and args.ip is None: - main_parser.print_help() - else: - if validate_ip_cidr(args.ip): - try: - for role in args.roles: - return_code = apply(role, args.ip) - if return_code > 0: - break - except Exception as e: - print(f'Unexpected exception occurred: {e}', file=sys.stderr) - return_code = e.errno + if os.environ.get('IP') is None: + main_parser.print_help() + sys.exit(1) else: - print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) - return_code = 1 - + args.ip = os.environ['IP'] + + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + sys.exit(return_code) From f93c6146f5ebe1a41fdb9addcb80d460d1e6e970 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 21 Oct 2021 15:24:55 -0400 Subject: [PATCH 039/110] docker binds requires --- salt/ca/init.sls | 5 +++-- salt/domainstats/init.sls | 4 +++- salt/elastalert/init.sls | 4 ++++ salt/elasticsearch/init.sls | 21 ++++++++++++++++++++- salt/ssl/init.sls | 3 +++ 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 485b01d3e..9c30b5c5e 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -24,8 +24,9 @@ pki_private_key: - x509: /etc/pki/ca.crt {%- endif %} -/etc/pki/ca.crt: +pki_public_ca_crt: x509.certificate_managed: + - name: /etc/pki/ca.crt - signing_private_key: /etc/pki/ca.key - CN: {{ manager }} - C: US @@ -66,4 +67,4 @@ cakeyperms: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/domainstats/init.sls b/salt/domainstats/init.sls index 72ccf2f76..225db7e72 100644 --- a/salt/domainstats/init.sls +++ b/salt/domainstats/init.sls @@ -53,6 +53,8 @@ so-domainstats: - user: domainstats - binds: - /opt/so/log/domainstats:/var/log/domain_stats + - require: + - file: dstatslogdir append_so-domainstats_so-status.conf: file.append: @@ -65,4 +67,4 @@ append_so-domainstats_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index a5c3a3b67..f94e0c1d0 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -122,6 +122,10 @@ so-elastalert: - {{MANAGER_URL}}:{{MANAGER_IP}} - require: - cmd: wait_for_elasticsearch + - file: elastarules + - file: elastalogdir + - file: elastacustmodulesdir + - file: elastaconf - watch: - file: elastaconf diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 9f475c2c3..f3908fd60 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,7 +15,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} - +include: + - ssl {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} @@ -280,6 +281,24 @@ so-elasticsearch: - file: esyml - file: esingestconf - file: so-elasticsearch-pipelines-file + - require: + - file: esyml + - file: eslog4jfile + - file: nsmesdir + - file: eslogdir + - file: cacertz + - file: /etc/pki/elasticsearch.crt + - file: /etc/pki/elasticsearch.key + - file: elasticp12perms + {% if ismanager %} + - file: pki_public_ca_crt + {% else %} + - file: trusttheca + {% endif %} + {% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} + - cmd: auth_users_roles_inode + - cmd: auth_users_inode + {% endif %} append_so-elasticsearch_so-status.conf: file.append: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 5690691fc..8ec404a5b 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -30,6 +30,9 @@ {% set ca_server = global_ca_server[0] %} {% endif %} +include: + - ca + # Trust the CA trusttheca: x509.pem_managed: From 451b19dc4d843286f1f6c890c36e117299527b5e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 22 Oct 2021 09:53:20 -0400 Subject: [PATCH 040/110] change from file to x509 --- salt/elasticsearch/init.sls | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index f3908fd60..41a9f4504 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -287,13 +287,13 @@ so-elasticsearch: - file: nsmesdir - file: eslogdir - file: cacertz - - file: /etc/pki/elasticsearch.crt - - file: /etc/pki/elasticsearch.key + - x509: /etc/pki/elasticsearch.crt + - x509: /etc/pki/elasticsearch.key - file: elasticp12perms {% if ismanager %} - - file: pki_public_ca_crt + - x509: pki_public_ca_crt {% else %} - - file: trusttheca + - x509: trusttheca {% endif %} {% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} - cmd: auth_users_roles_inode From 352e30f9e1509a23a0f64b8ae6ee0608085c9d38 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 Oct 2021 11:16:29 -0400 Subject: [PATCH 041/110] Add CUSTOM_FLEET_HOSTNAME to subjectAltName of fleet.key Resolves #4319 --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 5690691fc..9d4026b56 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -445,7 +445,7 @@ msslkeyperms: x509.certificate_managed: - signing_private_key: /etc/pki/fleet.key - CN: {{ manager }} - - subjectAltName: DNS:{{ manager }},IP:{{ managerip }} + - subjectAltName: DNS:{{ manager }},IP:{{ managerip }}{% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 62971d8c15ea29e58794d0c153e0d6fef2ced20c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 Oct 2021 11:57:47 -0400 Subject: [PATCH 042/110] Add Fleet custom hostname to end summary --- setup/so-whiptail | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index ed4067da1..3e4bf6151 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -505,6 +505,8 @@ whiptail_end_settings() { [[ -n $WEBUSER ]] && __append_end_msg "Web User: $WEBUSER" [[ -n $FLEETNODEUSER ]] && __append_end_msg "Fleet User: $FLEETNODEUSER" + + [[ -n $FLEETCUSTOMHOSTNAME ]] && __append_end_msg "Fleet Custom Hostname: $FLEETCUSTOMHOSTNAME" if [[ $is_manager ]]; then __append_end_msg "Enabled Optional Components:" From fed8bfac674e8c0434c61a627acc66f214400391 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 22 Oct 2021 14:10:59 -0400 Subject: [PATCH 043/110] more requires on docker containers --- salt/domainstats/init.sls | 3 +-- salt/filebeat/init.sls | 16 ++++++++++++---- salt/fleet/init.sls | 6 +++++- salt/grafana/init.sls | 2 ++ salt/influxdb/init.sls | 6 ++++++ salt/learn/logscan.sls | 2 ++ salt/logstash/init.sls | 16 ++++++++++++---- salt/ssl/init.sls | 36 +++++++++++++++++++++++------------- 8 files changed, 63 insertions(+), 24 deletions(-) diff --git a/salt/domainstats/init.sls b/salt/domainstats/init.sls index 225db7e72..e2167d161 100644 --- a/salt/domainstats/init.sls +++ b/salt/domainstats/init.sls @@ -45,8 +45,6 @@ so-domainstatsimage: so-domainstats: docker_container.running: - - require: - - so-domainstatsimage - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }} - hostname: domainstats - name: so-domainstats @@ -55,6 +53,7 @@ so-domainstats: - /opt/so/log/domainstats:/var/log/domain_stats - require: - file: dstatslogdir + - cmd: so-domainstatsimage append_so-domainstats_so-status.conf: file.append: diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index f03d3dc1a..ac45a50cd 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -25,9 +25,10 @@ {% from 'filebeat/map.jinja' import SO with context %} {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %} +include: + - ssl #only include elastic state for certain nodes {% if grains.role in ES_INCLUDED_NODES %} -include: - elasticsearch {% endif %} @@ -66,7 +67,7 @@ fileregistrydir: - makedirs: True # This needs to be owned by root -filebeatconfsync: +filebeatconf: file.managed: - name: /opt/so/conf/filebeat/etc/filebeat.yml - source: salt://filebeat/etc/filebeat.yml @@ -78,7 +79,7 @@ filebeatconfsync: OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} # Filebeat module config file -filebeatmoduleconfsync: +filebeatmoduleconf: file.managed: - name: /opt/so/conf/filebeat/etc/module-setup.yml - source: salt://filebeat/etc/module-setup.yml @@ -135,7 +136,14 @@ so-filebeat: {% endfor %} {% endfor %} - watch: - - file: /opt/so/conf/filebeat/etc/filebeat.yml + - file: filebeatconf + - require: + - file: filebeatconf + - file: filebeatmoduleconf + - file: filebeatmoduledir + - x509: filebeat_crt + - x509: filebeat_key + - x509: trusttheca {% if grains.role in ES_INCLUDED_NODES %} run_module_setup: diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 1bb4e73d6..2a0ac540b 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -17,6 +17,7 @@ include: + - ssl - mysql # Fleet Setup @@ -136,10 +137,13 @@ so-fleet: - /opt/so/conf/fleet/packs:/packs - watch: - /opt/so/conf/fleet/etc + - require: + - x509: fleet_key + - x509: fleet_crt append_so-fleet_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-fleet -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index 42df29af4..9113cdbe1 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -132,6 +132,8 @@ so-grafana: - 0.0.0.0:3000:3000 - watch: - file: /opt/so/conf/grafana/* + - require: + - file: grafana-config append_so-grafana_so-status.conf: file.append: diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index f270c9f73..218d2d18e 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -17,6 +17,8 @@ include: - salt.minion - salt.python3-influxdb + - ssl + # Influx DB influxconfdir: file.directory: @@ -60,6 +62,10 @@ so-influxdb: - 0.0.0.0:8086:8086 - watch: - file: influxdbconf + - require: + - file: influxdbconf + - x509: influxdb_key + - x509: influxdb_crt append_so-influxdb_so-status.conf: file.append: diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls index cc8bb2996..91f64420a 100644 --- a/salt/learn/logscan.sls +++ b/salt/learn/logscan.sls @@ -51,6 +51,8 @@ so-logscan: - /opt/so/log/logscan:/logscan/output:rw - /opt/so/log:/logscan/logs:ro - cpu_period: {{ logscan_cpu_period }} + - require: + - file: logscan_conf {% else %} - force: true {% endif %} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index cd6a8918c..50abd1e5b 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -45,6 +45,7 @@ {% endif %} include: + - ssl - elasticsearch # Create the logstash group @@ -162,9 +163,7 @@ so-logstash: {% endfor %} - binds: - /opt/so/conf/elasticsearch/templates/:/templates/:ro - - /opt/so/conf/logstash/etc/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro - - /opt/so/conf/logstash/etc/logstash.yml:/usr/share/logstash/config/logstash.yml:ro - - /opt/so/conf/logstash/etc/pipelines.yml:/usr/share/logstash/config/pipelines.yml + - /opt/so/conf/logstash/etc/:/usr/share/logstash/config/:ro - /opt/so/conf/logstash/pipelines:/usr/share/logstash/pipelines:ro - /opt/so/rules:/etc/nsm/rules:ro - /nsm/import:/nsm/import:ro @@ -181,7 +180,6 @@ so-logstash: {% endif %} - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro - - /etc/pki/ca.cer:/ca/ca.crt:ro {%- if grains['role'] == 'so-eval' %} - /nsm/zeek:/nsm/zeek:ro - /nsm/suricata:/suricata:ro @@ -201,6 +199,16 @@ so-logstash: {% for TEMPLATE in TEMPLATES %} - file: es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }} {% endfor %} + - require: + - x509: filebeat_crt + - x509: filebeat_key +{% if grains['role'] == 'so-heavynode' %} + - x509: trusttheca +{% else %} + - x509: pki_public_ca_crt +{% endif %} + - file: cacertz + - file: capemz append_so-logstash_so-status.conf: file.append: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 8ec404a5b..1e63a8980 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -67,8 +67,9 @@ removeesp12dir: - name: /etc/pki/elasticsearch.p12 - onlyif: "[ -d /etc/pki/elasticsearch.p12 ]" -/etc/pki/influxdb.key: +influxdb_key: x509.private_key_managed: + - name: /etc/pki/influxdb.key - CN: {{ manager }} - bits: 4096 - days_remaining: 0 @@ -85,8 +86,9 @@ removeesp12dir: interval: 30 # Create a cert for the talking to influxdb -/etc/pki/influxdb.crt: +influxdb_crt: x509.certificate_managed: + - name: /etc/pki/influxdb.crt - ca_server: {{ ca_server }} - signing_policy: influxdb - public_key: /etc/pki/influxdb.key @@ -161,8 +163,9 @@ rediskeyperms: {% endif %} {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %} -/etc/pki/filebeat.key: +filebeat_key: x509.private_key_managed: + - name: /etc/pki/filebeat.key - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 @@ -179,8 +182,9 @@ rediskeyperms: interval: 30 # Request a cert and drop it where it needs to go to be distributed -/etc/pki/filebeat.crt: +filebeat_crt: x509.certificate_managed: + - name: /etc/pki/filebeat.crt - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /etc/pki/filebeat.key @@ -201,7 +205,7 @@ rediskeyperms: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt" - onchanges: - - x509: /etc/pki/filebeat.key + - x509: filebeat_key fbperms: @@ -427,8 +431,9 @@ msslkeyperms: - group: 939 # Create a private key and cert for OSQuery -/etc/pki/fleet.key: +fleet_key: x509.private_key_managed: + - name: /etc/pki/fleet.key - CN: {{ manager }} - bits: 4096 - days_remaining: 0 @@ -444,8 +449,9 @@ msslkeyperms: attempts: 5 interval: 30 -/etc/pki/fleet.crt: +fleet_crt: x509.certificate_managed: + - name: /etc/pki/fleet.crt - signing_private_key: /etc/pki/fleet.key - CN: {{ manager }} - subjectAltName: DNS:{{ manager }},IP:{{ managerip }} @@ -476,8 +482,9 @@ fbcertdir: - name: /opt/so/conf/filebeat/etc/pki - makedirs: True -/opt/so/conf/filebeat/etc/pki/filebeat.key: +filebeat_key: x509.private_key_managed: + - name: /opt/so/conf/filebeat/etc/pki/filebeat.key - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 @@ -486,7 +493,7 @@ fbcertdir: - new: True {% if salt['file.file_exists']('/opt/so/conf/filebeat/etc/pki/filebeat.key') -%} - prereq: - - x509: /opt/so/conf/filebeat/etc/pki/filebeat.crt + - x509: filebeat_crt {%- endif %} - timeout: 30 - retry: @@ -494,8 +501,9 @@ fbcertdir: interval: 30 # Request a cert and drop it where it needs to go to be distributed -/opt/so/conf/filebeat/etc/pki/filebeat.crt: +filebeat_crt: x509.certificate_managed: + - name: /opt/so/conf/filebeat/etc/pki/filebeat.crt - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key @@ -519,7 +527,7 @@ filebeatpkcs: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /opt/so/conf/filebeat/etc/pki/filebeat.key -topk8 -out /opt/so/conf/filebeat/etc/pki/filebeat.p8 -passout pass:" - onchanges: - - x509: /opt/so/conf/filebeat/etc/pki/filebeat.key + - x509: filebeat_key filebeatkeyperms: file.managed: @@ -585,8 +593,9 @@ msslkeyperms: - group: 939 # Create a private key and cert for Fleet -/etc/pki/fleet.key: +fleet_key: x509.private_key_managed: + - name: /etc/pki/fleet.key - CN: {{ manager }} - bits: 4096 - days_remaining: 0 @@ -602,8 +611,9 @@ msslkeyperms: attempts: 5 interval: 30 -/etc/pki/fleet.crt: +fleet_crt: x509.certificate_managed: + - name: /etc/pki/fleet.crt - signing_private_key: /etc/pki/fleet.key - CN: {{ HOSTNAME }} - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }} {% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }} {% endif %} From f61400680dad58f92bef1bc6e1b7563b94cd35e8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 22 Oct 2021 14:22:15 -0400 Subject: [PATCH 044/110] fix dupe ids --- salt/filebeat/init.sls | 2 +- salt/logstash/init.sls | 1 - salt/ssl/init.sls | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index ac45a50cd..83f73de4f 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -142,7 +142,7 @@ so-filebeat: - file: filebeatmoduleconf - file: filebeatmoduledir - x509: filebeat_crt - - x509: filebeat_key + - x509: conf_filebeat_key - x509: trusttheca {% if grains.role in ES_INCLUDED_NODES %} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 50abd1e5b..a7ed361d6 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -201,7 +201,6 @@ so-logstash: {% endfor %} - require: - x509: filebeat_crt - - x509: filebeat_key {% if grains['role'] == 'so-heavynode' %} - x509: trusttheca {% else %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 1e63a8980..645ef0fe0 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -163,7 +163,7 @@ rediskeyperms: {% endif %} {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %} -filebeat_key: +etc_filebeat_key: x509.private_key_managed: - name: /etc/pki/filebeat.key - CN: {{ COMMONNAME }} @@ -205,7 +205,7 @@ filebeat_crt: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt" - onchanges: - - x509: filebeat_key + - x509: etc_filebeat_key fbperms: @@ -482,7 +482,7 @@ fbcertdir: - name: /opt/so/conf/filebeat/etc/pki - makedirs: True -filebeat_key: +conf_filebeat_key: x509.private_key_managed: - name: /opt/so/conf/filebeat/etc/pki/filebeat.key - CN: {{ COMMONNAME }} @@ -527,7 +527,7 @@ filebeatpkcs: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /opt/so/conf/filebeat/etc/pki/filebeat.key -topk8 -out /opt/so/conf/filebeat/etc/pki/filebeat.p8 -passout pass:" - onchanges: - - x509: filebeat_key + - x509: conf_filebeat_key filebeatkeyperms: file.managed: From 9f6407fcb0502df8e7ca9fd793de8c3cf81fdc02 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 22 Oct 2021 14:26:04 -0400 Subject: [PATCH 045/110] fix dupe ids --- salt/filebeat/init.sls | 2 +- salt/logstash/init.sls | 2 +- salt/ssl/init.sls | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 83f73de4f..2693663db 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -141,7 +141,7 @@ so-filebeat: - file: filebeatconf - file: filebeatmoduleconf - file: filebeatmoduledir - - x509: filebeat_crt + - x509: conf_filebeat_crt - x509: conf_filebeat_key - x509: trusttheca diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index a7ed361d6..c27631685 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -200,7 +200,7 @@ so-logstash: - file: es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }} {% endfor %} - require: - - x509: filebeat_crt + - x509: etc_filebeat_crt {% if grains['role'] == 'so-heavynode' %} - x509: trusttheca {% else %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 645ef0fe0..a741272d6 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -174,7 +174,7 @@ etc_filebeat_key: - new: True {% if salt['file.file_exists']('/etc/pki/filebeat.key') -%} - prereq: - - x509: /etc/pki/filebeat.crt + - x509: etc_filebeat_crt {%- endif %} - timeout: 30 - retry: @@ -182,7 +182,7 @@ etc_filebeat_key: interval: 30 # Request a cert and drop it where it needs to go to be distributed -filebeat_crt: +etc_filebeat_crt: x509.certificate_managed: - name: /etc/pki/filebeat.crt - ca_server: {{ ca_server }} @@ -493,7 +493,7 @@ conf_filebeat_key: - new: True {% if salt['file.file_exists']('/opt/so/conf/filebeat/etc/pki/filebeat.key') -%} - prereq: - - x509: filebeat_crt + - x509: conf_filebeat_crt {%- endif %} - timeout: 30 - retry: @@ -501,7 +501,7 @@ conf_filebeat_key: interval: 30 # Request a cert and drop it where it needs to go to be distributed -filebeat_crt: +conf_filebeat_crt: x509.certificate_managed: - name: /opt/so/conf/filebeat/etc/pki/filebeat.crt - ca_server: {{ ca_server }} From 283f7296bc59cabf5cc633e7c1a2bda228a74785 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 22 Oct 2021 14:45:22 -0400 Subject: [PATCH 046/110] fix require --- salt/filebeat/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 2693663db..75beb66c9 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -150,7 +150,7 @@ run_module_setup: cmd.run: - name: /usr/sbin/so-filebeat-module-setup - require: - - file: filebeatmoduleconfsync + - file: filebeatmoduleconf - docker_container: so-filebeat - onchanges: - docker_container: so-elasticsearch From 17af513692e8cf39e4206b485b67341c5ea96e85 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 Oct 2021 15:28:37 -0400 Subject: [PATCH 047/110] Escape single quotes and allow for any character in node description --- salt/sensoroni/files/sensoroni.json | 2 +- setup/so-functions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 1a6e6fc8c..743021a7d 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -17,7 +17,7 @@ "agent": { "nodeId": "{{ grains.host | lower }}", "role": "{{ grains.role }}", - "description": "{{ DESCRIPTION }}", + "description": {{ DESCRIPTION | tojson }}, "address": "{{ ADDRESS }}", "model": "{{ MODEL }}", "pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }}, diff --git a/setup/so-functions b/setup/so-functions index 58fbca562..62d458911 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1509,7 +1509,7 @@ host_pillar() { " mainint: '$MNIC'"\ "sensoroni:"\ " node_address: '$MAINIP'"\ - " node_description: '$NODE_DESCRIPTION'"\ + " node_description: '${NODE_DESCRIPTION//\'/''}'"\ "" > "$pillar_file" } From 38b16a507baa3c8cd2d00e16de768d7162c38471 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 Oct 2021 15:29:32 -0400 Subject: [PATCH 048/110] Update ip for root user in mysql when running so-ip-update --- salt/common/tools/sbin/so-ip-update | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-ip-update b/salt/common/tools/sbin/so-ip-update index 9976a42e8..8ef99fadc 100755 --- a/salt/common/tools/sbin/so-ip-update +++ b/salt/common/tools/sbin/so-ip-update @@ -8,9 +8,9 @@ fi echo "This tool will update a manager's IP address to the new IP assigned to the management network interface." -echo +echo "" echo "WARNING: This tool is still undergoing testing, use at your own risk!" -echo +echo "" if [ -z "$OLD_IP" ]; then OLD_IP=$(lookup_pillar "managerip") @@ -27,7 +27,7 @@ if [ -z "$NEW_IP" ]; then NEW_IP=$(ip -4 addr list $iface | grep inet | cut -d' ' -f6 | cut -d/ -f1) if [ -z "$NEW_IP" ]; then - fail "Unable to detect new IP on interface $iface. " + fail "Unable to detect new IP on interface $iface." fi echo "Detected new IP $NEW_IP on interface $iface." @@ -39,15 +39,20 @@ fi echo "About to change old IP $OLD_IP to new IP $NEW_IP." -echo +echo "" read -n 1 -p "Would you like to continue? (y/N) " CONTINUE -echo +echo "" if [ "$CONTINUE" == "y" ]; then - for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do - echo "Updating file: $file" - sed -i "s|$OLD_IP|$NEW_IP|g" $file - done + for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do + echo "Updating file: $file" + sed -i "s|$OLD_IP|$NEW_IP|g" $file + done + + echo "Granting MySQL root user permissions on $NEW_IP" + docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'$NEW_IP' IDENTIFIED BY '$(lookup_pillar_secret 'mysql')' WITH GRANT OPTION;" &> /dev/null + echo "Removing MySQL root user from $OLD_IP" + docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "DROP USER 'root'@'$OLD_IP';" &> /dev/null echo "The IP has been changed from $OLD_IP to $NEW_IP." @@ -60,4 +65,4 @@ if [ "$CONTINUE" == "y" ]; then fi else echo "Exiting without changes." -fi \ No newline at end of file +fi From d7e5377a449f80a90450b5cd1744eec61bb24ea7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 22 Oct 2021 16:46:45 -0400 Subject: [PATCH 049/110] more requires --- salt/manager/init.sls | 5 +++-- salt/minio/init.sls | 8 +++++++- salt/mysql/init.sls | 10 ++++++---- salt/nginx/init.sls | 11 ++++++++++- salt/pcap/init.sls | 4 +++- salt/redis/init.sls | 18 +++++++++++++----- salt/registry/init.sls | 4 ++++ salt/sensoroni/init.sls | 4 +++- salt/soctopus/init.sls | 8 +++++++- salt/ssl/init.sls | 30 ++++++++++++++++++++---------- salt/suricata/init.sls | 8 ++++++-- salt/telegraf/init.sls | 11 ++++++++++- salt/thehive/init.sls | 15 +++++++++++++-- salt/zeek/init.sls | 11 ++++++++--- 14 files changed, 113 insertions(+), 34 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 1d6577e5f..4a3769e1e 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -60,8 +60,7 @@ aptcacherlogdir: - group: 939 - makedirs: true -# Copy the config -acngcopyconf: +acngconf: file.managed: - name: /opt/so/conf/aptcacher-ng/etc/acng.conf - source: salt://manager/files/acng/acng.conf @@ -80,6 +79,8 @@ so-aptcacherng: - /opt/so/conf/aptcacher-ng/cache:/var/cache/apt-cacher-ng:rw - /opt/so/log/aptcacher-ng:/var/log/apt-cacher-ng:rw - /opt/so/conf/aptcacher-ng/etc/acng.conf:/etc/apt-cacher-ng/acng.conf:ro + - require: + - file: acngconf append_so-aptcacherng_so-status.conf: file.append: diff --git a/salt/minio/init.sls b/salt/minio/init.sls index f61209c8b..8e4d548f6 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -21,6 +21,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} +include: + - ssl + # Minio Setup minioconfdir: file.directory: @@ -59,6 +62,9 @@ so-minio: - /etc/pki/minio.key:/.minio/certs/private.key:ro - /etc/pki/minio.crt:/.minio/certs/public.crt:ro - entrypoint: "/usr/bin/docker-entrypoint.sh server --certs-dir /.minio/certs --address :9595 /data" + - require: + - file: minio_key + - file: minio_crt append_so-minio_so-status.conf: file.append: @@ -71,4 +77,4 @@ append_so-minio_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 46e62fcc0..b4c87eed4 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -45,13 +45,14 @@ mysqlpiddir: - group: 939 - makedirs: True -mysqletcsync: +mysqletc: file.recurse: - name: /opt/so/conf/mysql/etc - source: salt://mysql/etc - user: 939 - group: 939 - template: jinja + - mode: 640 mysqllogdir: file.directory: @@ -88,12 +89,13 @@ so-mysql: - MYSQL_ROOT_HOST={{ MAINIP }} - MYSQL_ROOT_PASSWORD=/etc/mypass - binds: - - /opt/so/conf/mysql/etc/my.cnf:/etc/my.cnf:ro - - /opt/so/conf/mysql/etc/mypass:/etc/mypass + - /opt/so/conf/mysql/etc/:/etc/:ro - /nsm/mysql:/var/lib/mysql:rw - /opt/so/log/mysql:/var/log/mysql:rw - watch: - /opt/so/conf/mysql/etc + - require: + - file: mysqletc cmd.run: - name: until nc -z {{ MAINIP }} 3306; do sleep 1; done - timeout: 600 @@ -118,4 +120,4 @@ append_so-mysql_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 15c1acc8e..12e0b1fdc 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -8,6 +8,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set ISAIRGAP = salt['pillar.get']('global:airgap') %} +include: + - ssl + # Drop the correct nginx config based on role nginxconfdir: file.directory: @@ -95,6 +98,12 @@ so-nginx: - watch: - file: nginxconf - file: nginxconfdir + - require: + - file: nginxconf + - x509: managerssl_key + - x509: managerssl_crt + - file: navigatorconfig + - file: navigatordefaultlayer append_so-nginx_so-status.conf: file.append: @@ -107,4 +116,4 @@ append_so-nginx_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 641300fdf..a29398db6 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -127,7 +127,9 @@ so-steno: - /nsm/pcaptmp:/tmp:rw - /opt/so/log/stenographer:/var/log/stenographer:rw - watch: - - file: /opt/so/conf/steno/config + - file: stenoconf + - require: + - file: stenoconf {% else %} {# if stenographer isn't enabled, then stop and remove the container #} - force: True {% endif %} diff --git a/salt/redis/init.sls b/salt/redis/init.sls index a99df219f..d52c49d5b 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -19,6 +19,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} +include: + - ssl + # Redis Setup redisconfdir: file.directory: @@ -41,10 +44,10 @@ redislogdir: - group: 939 - makedirs: True -redisconfsync: - file.recurse: - - name: /opt/so/conf/redis/etc - - source: salt://redis/etc +redisconf: + file.managed: + - name: /opt/so/conf/redis/etc/redis.conf + - source: salt://redis/etc/redis.conf - user: 939 - group: 939 - template: jinja @@ -67,6 +70,11 @@ so-redis: - entrypoint: "redis-server /usr/local/etc/redis/redis.conf" - watch: - file: /opt/so/conf/redis/etc + - require: + - file: redisconf + - x509: redis_crt + - x509: redis_key + - x509: pki_public_ca_crt append_so-redis_so-status.conf: file.append: @@ -79,4 +87,4 @@ append_so-redis_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/registry/init.sls b/salt/registry/init.sls index eb0c2df0c..b89bd5dff 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -47,6 +47,10 @@ so-dockerregistry: - retry: attempts: 5 interval: 30 + - require: + - file: dockerregistryconf + - x509: registry_crt + - x509: registry_key append_so-dockerregistry_so-status.conf: file.append: diff --git a/salt/sensoroni/init.sls b/salt/sensoroni/init.sls index a55049c06..1405c72bf 100644 --- a/salt/sensoroni/init.sls +++ b/salt/sensoroni/init.sls @@ -38,8 +38,10 @@ so-sensoroni: - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw - watch: - file: /opt/so/conf/sensoroni/sensoroni.json + - require: + - file: sensoroniagentconf append_so-sensoroni_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - - text: so-sensoroni \ No newline at end of file + - text: so-sensoroni diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 724e5a617..576cc573d 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -8,6 +8,9 @@ {% set MANAGER_IP = salt['pillar.get']('global:managerip', '') %} {% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} +include: + - nginx + soctopusdir: file.directory: - name: /opt/so/conf/soctopus/sigma-import @@ -71,6 +74,9 @@ so-soctopus: - 0.0.0.0:7000:7000 - extra_hosts: - {{MANAGER_URL}}:{{MANAGER_IP}} + - require: + - file: soctopusconf + - file: navigatordefaultlayer append_so-soctopus_so-status.conf: file.append: @@ -83,4 +89,4 @@ append_so-soctopus_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index a741272d6..e18c71fc5 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -117,8 +117,9 @@ influxkeyperms: {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet'] %} # Create a cert for Redis encryption -/etc/pki/redis.key: +redis_key: x509.private_key_managed: + - name: /etc/pki/redis.key - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 @@ -134,8 +135,9 @@ influxkeyperms: attempts: 5 interval: 30 -/etc/pki/redis.crt: +redis_crt: x509.certificate_managed: + - name: /etc/pki/redis.crt - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/redis.key @@ -244,8 +246,9 @@ fbcrtlink: - user: socore - group: socore -/etc/pki/registry.key: +registry_key: x509.private_key_managed: + - name: /etc/pki/registry.key - CN: {{ manager }} - bits: 4096 - days_remaining: 0 @@ -262,8 +265,9 @@ fbcrtlink: interval: 30 # Create a cert for the docker registry -/etc/pki/registry.crt: +registry_crt: x509.certificate_managed: + - name: /etc/pki/registry.crt - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/registry.key @@ -287,8 +291,9 @@ regkeyperms: - mode: 640 - group: 939 -/etc/pki/minio.key: +minio_key: x509.private_key_managed: + - name: /etc/pki/minio.key - CN: {{ manager }} - bits: 4096 - days_remaining: 0 @@ -305,8 +310,9 @@ regkeyperms: interval: 30 # Create a cert for minio -/etc/pki/minio.crt: +minio_crt: x509.certificate_managed: + - name: /etc/pki/minio.crt - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/minio.key @@ -386,8 +392,9 @@ elasticp12perms: - mode: 640 - group: 930 -/etc/pki/managerssl.key: +managerssl_key: x509.private_key_managed: + - name: /etc/pki/managerssl.key - CN: {{ manager }} - bits: 4096 - days_remaining: 0 @@ -404,8 +411,9 @@ elasticp12perms: interval: 30 # Create a cert for the reverse proxy -/etc/pki/managerssl.crt: +managerssl_crt: x509.certificate_managed: + - name: /etc/pki/managerssl.crt - ca_server: {{ ca_server }} - signing_policy: managerssl - public_key: /etc/pki/managerssl.key @@ -548,8 +556,9 @@ chownfilebeatp8: {% if grains['role'] == 'so-fleet' %} -/etc/pki/managerssl.key: +managerssl_key: x509.private_key_managed: + - name: /etc/pki/managerssl.key - CN: {{ manager }} - bits: 4096 - days_remaining: 0 @@ -566,8 +575,9 @@ chownfilebeatp8: interval: 30 # Create a cert for the reverse proxy -/etc/pki/managerssl.crt: +managerssl_crt: x509.certificate_managed: + - name: /etc/pki/managerssl.crt - ca_server: {{ ca_server }} - signing_policy: managerssl - public_key: /etc/pki/managerssl.key diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 73c4d2395..f83f54c5c 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -93,7 +93,7 @@ surilogscript: - month: '*' - dayweek: '*' -suriconfigsync: +suriconfig: file.managed: - name: /opt/so/conf/suricata/suricata.yaml - source: salt://suricata/files/suricata.yaml.jinja @@ -155,10 +155,14 @@ so-suricata: - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro - network_mode: host - watch: - - file: /opt/so/conf/suricata/suricata.yaml + - file: suriconfig - file: surithresholding - file: /opt/so/conf/suricata/rules/ - file: /opt/so/conf/suricata/bpf + - require: + - file: suriconfig + - file: surithresholding + - file: suribpf {% else %} {# if Suricata isn't enabled, then stop and remove the container #} - force: True diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index 615cfc237..d71916430 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -88,7 +88,16 @@ so-telegraf: - file: tgrafconf - file: tgrafsyncscripts - file: node_config - + - require: + - file: tgrafconf + - file: node_config + {% if grains['role'] == 'so-manager' or grains['role'] == 'so-eval' or grains['role'] == 'so-managersearch' %} + - x509: pki_public_ca_crt + {% else %} + - x509: trusttheca + {% endif %} + - x509: influxdb_crt + - x509: influxdb_key append_so-telegraf_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/thehive/init.sls b/salt/thehive/init.sls index d1ee8a4bf..61a7a053c 100644 --- a/salt/thehive/init.sls +++ b/salt/thehive/init.sls @@ -83,8 +83,7 @@ so-thehive-es: - tty: True - binds: - /nsm/thehive/esdata:/usr/share/elasticsearch/data:rw - - /opt/so/conf/thehive/etc/es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro - - /opt/so/conf/thehive/etc/es/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro + - /opt/so/conf/thehive/etc/es/:/usr/share/elasticsearch/config/:ro - /opt/so/log/thehive:/var/log/elasticsearch:rw - environment: - ES_JAVA_OPTS=-Xms512m -Xmx512m @@ -97,6 +96,14 @@ append_so-thehive-es_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-thehive-es +cortex_application_conf: + file.exists: + - name: /opt/so/conf/thehive/etc/cortex-application.conf + +application_conf: + file.exists: + - name: /opt/so/conf/thehive/etc/application.conf + # Install Cortex so-cortex: docker_container.running: @@ -110,6 +117,8 @@ so-cortex: - /opt/so/conf/cortex/custom-responders:/custom-responders:ro - port_bindings: - 0.0.0.0:9001:9001 + - require: + - file: cortex_application_conf append_so-cortex_so-status.conf: file.append: @@ -135,6 +144,8 @@ so-thehive: - /opt/so/conf/thehive/etc/application.conf:/opt/thehive/conf/application.conf:ro - port_bindings: - 0.0.0.0:9000:9000 + - require: + - file: application_conf append_so-thehive_so-status.conf: file.append: diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 5f5adfaa6..e4b83a9e1 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -116,7 +116,7 @@ zeekctlcfg: ZEEKCTL: {{ ZEEK.zeekctl | tojson }} # Sync node.cfg -nodecfgsync: +nodecfg: file.managed: - name: /opt/so/conf/zeek/node.cfg - source: salt://zeek/files/node.cfg @@ -182,7 +182,7 @@ zeekbpf: {% endif %} -localzeeksync: +localzeek: file.managed: - name: /opt/so/conf/zeek/local.zeek - source: salt://zeek/files/local.zeek.jinja @@ -219,6 +219,11 @@ so-zeek: - file: /opt/so/conf/zeek/zeekctl.cfg - file: /opt/so/conf/zeek/policy - file: /opt/so/conf/zeek/bpf + - require: + - file: localzeek + - file: nodecfg + - file: zeekctlcfg + - file: zeekbpf {% else %} {# if Zeek isn't enabled, then stop and remove the container #} - force: True {% endif %} @@ -247,4 +252,4 @@ delete_so-zeek_so-status.disabled: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From ee7e714f435688b1db66b2e6f12ff41ed0b703dd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 22 Oct 2021 16:55:23 -0400 Subject: [PATCH 050/110] change to file_mode --- salt/mysql/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index b4c87eed4..d958f55de 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -52,7 +52,7 @@ mysqletc: - user: 939 - group: 939 - template: jinja - - mode: 640 + - file_mode: 640 mysqllogdir: file.directory: From 6e34905b4291edc7db3e329483695be60c4c818d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 Oct 2021 15:28:37 -0400 Subject: [PATCH 051/110] Escape single quotes and allow for any character in node description --- salt/sensoroni/files/sensoroni.json | 2 +- setup/so-functions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 1a6e6fc8c..743021a7d 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -17,7 +17,7 @@ "agent": { "nodeId": "{{ grains.host | lower }}", "role": "{{ grains.role }}", - "description": "{{ DESCRIPTION }}", + "description": {{ DESCRIPTION | tojson }}, "address": "{{ ADDRESS }}", "model": "{{ MODEL }}", "pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }}, diff --git a/setup/so-functions b/setup/so-functions index 58fbca562..62d458911 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1509,7 +1509,7 @@ host_pillar() { " mainint: '$MNIC'"\ "sensoroni:"\ " node_address: '$MAINIP'"\ - " node_description: '$NODE_DESCRIPTION'"\ + " node_description: '${NODE_DESCRIPTION//\'/''}'"\ "" > "$pillar_file" } From 7fa43a276a1b86772f13b0a21c19343ae1910372 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Oct 2021 13:15:20 -0400 Subject: [PATCH 052/110] Rename default headers and host for HTTP input --- salt/logstash/pipelines/config/so/0011_input_endgame.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/logstash/pipelines/config/so/0011_input_endgame.conf b/salt/logstash/pipelines/config/so/0011_input_endgame.conf index b87d8e9b2..375585957 100644 --- a/salt/logstash/pipelines/config/so/0011_input_endgame.conf +++ b/salt/logstash/pipelines/config/so/0011_input_endgame.conf @@ -3,6 +3,8 @@ input { id => "endgame_data" port => 3765 codec => es_bulk + request_headers_target_field => client_headers + remote_host_target_field => client_host ssl => true ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] ssl_certificate => "/usr/share/logstash/filebeat.crt" From 3be0d05eeab7b6cf714fbabf58c34f2ab1a6c00e Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Oct 2021 13:16:30 -0400 Subject: [PATCH 053/110] Update field removal based on HTTP input changes --- .../logstash/pipelines/config/so/9900_output_endgame.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja b/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja index f23913637..b5920fe40 100644 --- a/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja +++ b/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja @@ -8,7 +8,7 @@ filter { if [event][module] =~ "endgame" { mutate { - remove_field => ["headers", "host"] + remove_field => ["client_headers", "client_host"] } } } From d5f42e0d7c8eac51ac4789dfcc5a9da9add6a3ef Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 25 Oct 2021 15:06:42 -0400 Subject: [PATCH 054/110] Update whiptail links to use latest docs --- setup/so-whiptail | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index ed4067da1..e74529438 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -285,7 +285,7 @@ whiptail_storage_requirements() { You need ${needed_val} to meet minimum requirements. - Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. + Visit https://docs.securityonion.net/en/latest/hardware.html for more information. Select YES to continue anyway, or select NO to cancel. EOM @@ -1774,7 +1774,7 @@ whiptail_storage_requirements() { You need ${needed_val} to meet minimum requirements. - Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. + Visit https://docs.securityonion.net/en/latest/hardware.html for more information. Press YES to continue anyway, or press NO to cancel. EOM From 9a78d13bee022122e5100f2f5a7bd6927cfeba9c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 25 Oct 2021 15:37:23 -0400 Subject: [PATCH 055/110] change perms on mysql --- salt/mysql/etc/mypass | 1 - salt/mysql/init.sls | 24 ++++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/salt/mysql/etc/mypass b/salt/mysql/etc/mypass index f5f781c10..b38bf75ec 100644 --- a/salt/mysql/etc/mypass +++ b/salt/mysql/etc/mypass @@ -1,2 +1 @@ -{%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) -%} {{ MYSQLPASS }} diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index d958f55de..750ae1c5e 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -45,14 +45,24 @@ mysqlpiddir: - group: 939 - makedirs: True -mysqletc: - file.recurse: - - name: /opt/so/conf/mysql/etc - - source: salt://mysql/etc +mysqlcnf: + file.managed: + - name: /opt/so/conf/mysql/etc/my.cnf + - source: salt://mysql/etc/my.cnf + - user: 939 + - group: 939 + - file_mode: 640 + +mysqlpass: + file.managed: + - name: /opt/so/conf/mysql/etc/mypass + - source: salt://mysql/etc/mypass - user: 939 - group: 939 - template: jinja - file_mode: 640 + - defaults: + MYSQLPASS: {{ MYSQLPASS }} mysqllogdir: file.directory: @@ -89,13 +99,15 @@ so-mysql: - MYSQL_ROOT_HOST={{ MAINIP }} - MYSQL_ROOT_PASSWORD=/etc/mypass - binds: - - /opt/so/conf/mysql/etc/:/etc/:ro + - /opt/so/conf/mysql/etc/my.cnf:/etc/my.cnf:ro + - /opt/so/conf/mysql/etc/mypass:/etc/mypass - /nsm/mysql:/var/lib/mysql:rw - /opt/so/log/mysql:/var/log/mysql:rw - watch: - /opt/so/conf/mysql/etc - require: - - file: mysqletc + - file: mysqlcnf + - file: mysqlpass cmd.run: - name: until nc -z {{ MAINIP }} 3306; do sleep 1; done - timeout: 600 From 7e8d74e770bfffbc32192ed9cd81a864182de6bc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 25 Oct 2021 15:50:27 -0400 Subject: [PATCH 056/110] just use mode --- salt/mysql/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 750ae1c5e..54523ad17 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -51,7 +51,7 @@ mysqlcnf: - source: salt://mysql/etc/my.cnf - user: 939 - group: 939 - - file_mode: 640 + - mode: 640 mysqlpass: file.managed: @@ -60,7 +60,7 @@ mysqlpass: - user: 939 - group: 939 - template: jinja - - file_mode: 640 + - mode: 640 - defaults: MYSQLPASS: {{ MYSQLPASS }} From fa2edb2b592c66f069d18a13971fd67204d5b22d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 26 Oct 2021 08:39:30 -0400 Subject: [PATCH 057/110] make cortex_init and hive_init time out after 1 minutes vs 5 minutes --- salt/thehive/scripts/cortex_init | 4 ++-- salt/thehive/scripts/hive_init | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index c2f00390d..d2002231b 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -29,7 +29,7 @@ cortex_init(){ CORTEX_ORG_USER_KEY="{{CORTEXORGUSERKEY}}" SOCTOPUS_CONFIG="$default_salt_dir/salt/soctopus/files/SOCtopus.conf" - if wait_for_web_response $CORTEX_URL "Cortex"; then + if wait_for_web_response $CORTEX_URL "Cortex" 60; then # Migrate DB curl -sk -XPOST -L "$CORTEX_API_URL/maintenance/migrate" @@ -65,7 +65,7 @@ if [ -f /opt/so/state/cortex.txt ]; then cortex_clean exit 0 else - if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"'; then + if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"' 60; then cortex_init cortex_clean else diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index ca6f7aa2b..454d484dd 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -20,7 +20,7 @@ thehive_init(){ SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf" echo -n "Waiting for TheHive..." - if wait_for_web_response $THEHIVE_URL "TheHive"; then + if wait_for_web_response $THEHIVE_URL "TheHive" 60; then # Migrate DB curl -sk -XPOST -L "$THEHIVE_API_URL/maintenance/migrate" @@ -43,7 +43,7 @@ if [ -f /opt/so/state/thehive.txt ]; then thehive_clean exit 0 else - if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"'; then + if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"' 60; then thehive_init thehive_clean else From 682cbfd223f3c4792f37b4e009a7bbf4f96844c2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 26 Oct 2021 09:23:24 -0400 Subject: [PATCH 058/110] remove the mode --- salt/mysql/init.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 54523ad17..cb9586984 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -51,7 +51,6 @@ mysqlcnf: - source: salt://mysql/etc/my.cnf - user: 939 - group: 939 - - mode: 640 mysqlpass: file.managed: @@ -60,7 +59,6 @@ mysqlpass: - user: 939 - group: 939 - template: jinja - - mode: 640 - defaults: MYSQLPASS: {{ MYSQLPASS }} From d77328608ec5b750a531c879f893518ed145fb59 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 26 Oct 2021 10:23:33 -0400 Subject: [PATCH 059/110] Remove port bindings for steno Steno runs in host mode so port bindings are not required --- salt/pcap/init.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 641300fdf..44e7323ad 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -117,8 +117,6 @@ so-steno: - start: {{ STENOOPTIONS.start }} - network_mode: host - privileged: True - - port_bindings: - - 127.0.0.1:1234:1234 - binds: - /opt/so/conf/steno/certs:/etc/stenographer/certs:rw - /opt/so/conf/steno/config:/etc/stenographer/config:rw From 36ae07b78edfbdfba3e4a2526b545a2b91d5f017 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 26 Oct 2021 10:49:50 -0400 Subject: [PATCH 060/110] change timeout from 60 to 120 --- salt/thehive/scripts/cortex_init | 4 ++-- salt/thehive/scripts/hive_init | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index d2002231b..fd0387131 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -29,7 +29,7 @@ cortex_init(){ CORTEX_ORG_USER_KEY="{{CORTEXORGUSERKEY}}" SOCTOPUS_CONFIG="$default_salt_dir/salt/soctopus/files/SOCtopus.conf" - if wait_for_web_response $CORTEX_URL "Cortex" 60; then + if wait_for_web_response $CORTEX_URL "Cortex" 120; then # Migrate DB curl -sk -XPOST -L "$CORTEX_API_URL/maintenance/migrate" @@ -65,7 +65,7 @@ if [ -f /opt/so/state/cortex.txt ]; then cortex_clean exit 0 else - if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"' 60; then + if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"' 120; then cortex_init cortex_clean else diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index 454d484dd..7ace6137b 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -20,7 +20,7 @@ thehive_init(){ SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf" echo -n "Waiting for TheHive..." - if wait_for_web_response $THEHIVE_URL "TheHive" 60; then + if wait_for_web_response $THEHIVE_URL "TheHive" 120; then # Migrate DB curl -sk -XPOST -L "$THEHIVE_API_URL/maintenance/migrate" @@ -43,7 +43,7 @@ if [ -f /opt/so/state/thehive.txt ]; then thehive_clean exit 0 else - if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"' 60; then + if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"' 120; then thehive_init thehive_clean else From c0dd9efd9b485ad2c0a5c0e6d065461cea531a7b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 26 Oct 2021 10:50:16 -0400 Subject: [PATCH 061/110] change so-thehive-es binds and requires --- salt/thehive/init.sls | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/thehive/init.sls b/salt/thehive/init.sls index 61a7a053c..d7050e226 100644 --- a/salt/thehive/init.sls +++ b/salt/thehive/init.sls @@ -73,6 +73,14 @@ thehiveesdata: - user: 939 - group: 939 +thehive_elasticsearch_yml: + file.exists: + - name: /opt/so/conf/thehive/etc/es/elasticsearch.yml + +log4j2_properties: + file.exists: + - name: /opt/so/conf/thehive/etc/es/log4j2.properties + so-thehive-es: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }} @@ -83,13 +91,17 @@ so-thehive-es: - tty: True - binds: - /nsm/thehive/esdata:/usr/share/elasticsearch/data:rw - - /opt/so/conf/thehive/etc/es/:/usr/share/elasticsearch/config/:ro + - /opt/so/conf/thehive/etc/es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + - /opt/so/conf/thehive/etc/es/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro - /opt/so/log/thehive:/var/log/elasticsearch:rw - environment: - ES_JAVA_OPTS=-Xms512m -Xmx512m - port_bindings: - 0.0.0.0:9400:9400 - 0.0.0.0:9500:9500 + - require: + - file: thehive_elasticsearch_yml + - file: log4j2_properties append_so-thehive-es_so-status.conf: file.append: From 58d62f29eac4d3cc500c810fb32d2d472e17fe50 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 26 Oct 2021 11:55:47 -0400 Subject: [PATCH 062/110] include ssl state in registry state --- salt/registry/init.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/registry/init.sls b/salt/registry/init.sls index b89bd5dff..76ccbf070 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -1,6 +1,9 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +include: + - ssl + # Create the config directory for the docker registry dockerregistryconfdir: file.directory: From bf403a8307ffab7bd964312bab0a73824305cdf2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 27 Oct 2021 09:47:12 -0400 Subject: [PATCH 063/110] only manager nodes get cert, key and att&ck binds --- salt/nginx/init.sls | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 12e0b1fdc..c732db636 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -76,34 +76,38 @@ so-nginx: - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw + - /opt/so/conf/fleet/packages:/opt/socore/html/packages + {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %} - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro - - /opt/so/conf/fleet/packages:/opt/socore/html/packages - {% if ISAIRGAP is sameas true %} - - /nsm/repo:/opt/socore/html/repo:ro - {% endif %} # ATT&CK Navigator binds - /opt/so/conf/navigator/navigator_config.json:/opt/socore/html/navigator/assets/config.json:ro - /opt/so/conf/navigator/nav_layer_playbook.json:/opt/socore/html/navigator/assets/playbook.json:ro + {% endif %} + {% if ISAIRGAP is sameas true %} + - /nsm/repo:/opt/socore/html/repo:ro + {% endif %} - cap_add: NET_BIND_SERVICE - port_bindings: - 80:80 - 443:443 - {% if ISAIRGAP is sameas true %} + {% if ISAIRGAP is sameas true %} - 7788:7788 - {% endif %} - {%- if FLEETMANAGER or FLEETNODE %} + {% endif %} + {%- if FLEETMANAGER or FLEETNODE %} - 8090:8090 - {%- endif %} + {%- endif %} - watch: - file: nginxconf - file: nginxconfdir - require: - file: nginxconf + {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %} - x509: managerssl_key - x509: managerssl_crt - file: navigatorconfig - file: navigatordefaultlayer + {% endif %} append_so-nginx_so-status.conf: file.append: From 90d473f2d6de1d9ff0a9fc07d7a9b4557c185cfe Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 Oct 2021 15:29:32 -0400 Subject: [PATCH 064/110] Update ip for root user in mysql when running so-ip-update --- salt/common/tools/sbin/so-ip-update | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-ip-update b/salt/common/tools/sbin/so-ip-update index 9976a42e8..8ef99fadc 100755 --- a/salt/common/tools/sbin/so-ip-update +++ b/salt/common/tools/sbin/so-ip-update @@ -8,9 +8,9 @@ fi echo "This tool will update a manager's IP address to the new IP assigned to the management network interface." -echo +echo "" echo "WARNING: This tool is still undergoing testing, use at your own risk!" -echo +echo "" if [ -z "$OLD_IP" ]; then OLD_IP=$(lookup_pillar "managerip") @@ -27,7 +27,7 @@ if [ -z "$NEW_IP" ]; then NEW_IP=$(ip -4 addr list $iface | grep inet | cut -d' ' -f6 | cut -d/ -f1) if [ -z "$NEW_IP" ]; then - fail "Unable to detect new IP on interface $iface. " + fail "Unable to detect new IP on interface $iface." fi echo "Detected new IP $NEW_IP on interface $iface." @@ -39,15 +39,20 @@ fi echo "About to change old IP $OLD_IP to new IP $NEW_IP." -echo +echo "" read -n 1 -p "Would you like to continue? (y/N) " CONTINUE -echo +echo "" if [ "$CONTINUE" == "y" ]; then - for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do - echo "Updating file: $file" - sed -i "s|$OLD_IP|$NEW_IP|g" $file - done + for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do + echo "Updating file: $file" + sed -i "s|$OLD_IP|$NEW_IP|g" $file + done + + echo "Granting MySQL root user permissions on $NEW_IP" + docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'$NEW_IP' IDENTIFIED BY '$(lookup_pillar_secret 'mysql')' WITH GRANT OPTION;" &> /dev/null + echo "Removing MySQL root user from $OLD_IP" + docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "DROP USER 'root'@'$OLD_IP';" &> /dev/null echo "The IP has been changed from $OLD_IP to $NEW_IP." @@ -60,4 +65,4 @@ if [ "$CONTINUE" == "y" ]; then fi else echo "Exiting without changes." -fi \ No newline at end of file +fi From e7f43cff5e3749636f6e6fb6263bceccf9af2187 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 27 Oct 2021 10:45:10 -0400 Subject: [PATCH 065/110] limit nodes that bind filebeat certs in so-logstash --- salt/logstash/init.sls | 102 +++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index c27631685..069b2f7bd 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -15,34 +15,34 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} -{% set MANAGER = salt['grains.get']('master') %} -{% set MANAGERIP = salt['pillar.get']('global:managerip') %} + {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} + {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} + {% set MANAGER = salt['grains.get']('master') %} + {% set MANAGERIP = salt['pillar.get']('global:managerip') %} -# Logstash Section - Decide which pillar to use -{% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %} -{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} - {% set freq = salt['pillar.get']('manager:freq', '0') %} - {% set dstats = salt['pillar.get']('manager:domainstats', '0') %} - {% set nodetype = salt['grains.get']('role', '') %} -{% elif grains['role'] == 'so-helix' %} - {% set freq = salt['pillar.get']('manager:freq', '0') %} - {% set dstats = salt['pillar.get']('manager:domainstats', '0') %} - {% set nodetype = salt['grains.get']('role', '') %} -{% endif %} + # Logstash Section - Decide which pillar to use + {% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %} + {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} + {% set freq = salt['pillar.get']('manager:freq', '0') %} + {% set dstats = salt['pillar.get']('manager:domainstats', '0') %} + {% set nodetype = salt['grains.get']('role', '') %} + {% elif grains['role'] == 'so-helix' %} + {% set freq = salt['pillar.get']('manager:freq', '0') %} + {% set dstats = salt['pillar.get']('manager:domainstats', '0') %} + {% set nodetype = salt['grains.get']('role', '') %} + {% endif %} -{% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} -{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} + {% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} + {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} + {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} -{% if grains.role in ['so-heavynode'] %} - {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %} - {% set EXTRAHOSTIP = salt['pillar.get']('sensor:mainip') %} -{% else %} - {% set EXTRAHOSTHOSTNAME = MANAGER %} - {% set EXTRAHOSTIP = MANAGERIP %} -{% endif %} + {% if grains.role in ['so-heavynode'] %} + {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %} + {% set EXTRAHOSTIP = salt['pillar.get']('sensor:mainip') %} + {% else %} + {% set EXTRAHOSTHOSTNAME = MANAGER %} + {% set EXTRAHOSTIP = MANAGERIP %} + {% endif %} include: - ssl @@ -74,22 +74,22 @@ lspipelinedir: - user: 931 - group: 939 -{% for PL in PIPELINES %} - {% for CONFIGFILE in PIPELINES[PL].config %} + {% for PL in PIPELINES %} + {% for CONFIGFILE in PIPELINES[PL].config %} ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: - source: salt://logstash/pipelines/config/{{CONFIGFILE}} - {% if 'jinja' in CONFIGFILE.split('.')[-1] %} + {% if 'jinja' in CONFIGFILE.split('.')[-1] %} - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} - template: jinja - {% else %} + {% else %} - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}} - {% endif %} + {% endif %} - user: 931 - group: 939 - mode: 660 - makedirs: True - {% endfor %} + {% endfor %} ls_pipeline_{{PL}}: file.directory: @@ -97,12 +97,12 @@ ls_pipeline_{{PL}}: - user: 931 - group: 939 - require: - {% for CONFIGFILE in PIPELINES[PL].config %} + {% for CONFIGFILE in PIPELINES[PL].config %} - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} - {% endfor %} + {% endfor %} - clean: True -{% endfor %} + {% endfor %} lspipelinesyml: file.managed: @@ -158,9 +158,9 @@ so-logstash: - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} - port_bindings: -{% for BINDING in DOCKER_OPTIONS.port_bindings %} + {% for BINDING in DOCKER_OPTIONS.port_bindings %} - {{ BINDING }} -{% endfor %} + {% endfor %} - binds: - /opt/so/conf/elasticsearch/templates/:/templates/:ro - /opt/so/conf/logstash/etc/:/usr/share/logstash/config/:ro @@ -170,42 +170,46 @@ so-logstash: - /nsm/logstash:/usr/share/logstash/data:rw - /opt/so/log/logstash:/var/log/logstash:rw - /sys/fs/cgroup:/sys/fs/cgroup:ro + {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %} - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro + {% endif %} - /opt/so/conf/logstash/etc/certs:/usr/share/logstash/certs:ro - {% if grains['role'] == 'so-heavynode' %} + {% if grains['role'] == 'so-heavynode' %} - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro - {% else %} + {% else %} - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro - {% endif %} + {% endif %} - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro - {%- if grains['role'] == 'so-eval' %} + {%- if grains['role'] == 'so-eval' %} - /nsm/zeek:/nsm/zeek:ro - /nsm/suricata:/suricata:ro - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro - /nsm/wazuh/logs/archives:/wazuh/archives:ro - /opt/so/log/fleet/:/osquery/logs:ro - /opt/so/log/strelka:/strelka:ro - {%- endif %} + {%- endif %} - watch: - file: lsetcsync -{% for PL in PIPELINES %} + {% for PL in PIPELINES %} - file: ls_pipeline_{{PL}} - {% for CONFIGFILE in PIPELINES[PL].config %} + {% for CONFIGFILE in PIPELINES[PL].config %} - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% endfor %} {% endfor %} -{% endfor %} -{% for TEMPLATE in TEMPLATES %} + {% for TEMPLATE in TEMPLATES %} - file: es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }} -{% endfor %} + {% endfor %} - require: + {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %} - x509: etc_filebeat_crt -{% if grains['role'] == 'so-heavynode' %} + {% endif %} + {% if grains['role'] == 'so-heavynode' %} - x509: trusttheca -{% else %} + {% else %} - x509: pki_public_ca_crt -{% endif %} + {% endif %} - file: cacertz - file: capemz From 9e9079f9cb76144658e15bf357488a8a248735c8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 27 Oct 2021 11:03:00 -0400 Subject: [PATCH 066/110] Reorder airgap prompt and add additional logic Setup should now only ask the user whether to setup as airgap on manager-type installs. For all distributed minions setup will now inherit the airgap boolean from the manager. --- setup/so-setup | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 119a0d2ff..982ce5f05 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -265,14 +265,6 @@ if [[ $is_manager || $is_import ]]; then fi if ! [[ -f $install_opt_file ]]; then - # Check if this is an airgap install - if [[ ( $is_manager || $is_import || $is_minion ) && $is_iso ]]; then - whiptail_airgap - if [[ "$INTERWEBS" == 'AIRGAP' ]]; then - is_airgap=true - fi - fi - if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone" elif [[ $is_fleet_standalone ]]; then @@ -312,17 +304,6 @@ if ! [[ -f $install_opt_file ]]; then add_mngr_ip_to_hosts fi - reset_proxy - if [[ -z $is_airgap ]]; then - collect_net_method - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 - fi - - if [[ $is_minion ]]; then - whiptail_ssh_key_copy_notice - copy_ssh_key >> $setup_log 2>&1 - fi - if [[ $is_minion ]] && ! (compare_versions); then info "Installer version mismatch, downloading correct version from manager" printf '%s\n' \ @@ -331,17 +312,36 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ "MSRVIP=$MSRVIP" \ - "is_airgap=$is_airgap" \ "NODE_DESCRIPTION=\"$NODE_DESCRIPTION\"" > "$install_opt_file" - [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi - else rm -rf $install_opt_file >> "$setup_log" 2>&1 fi +# Check if this is an airgap install +if [[ ( $is_manager || $is_import) && $is_iso ]]; then + whiptail_airgap + if [[ "$INTERWEBS" == 'AIRGAP' ]]; then + is_airgap=true + fi +el [[ $is_minion ]] + airgap_mngr_str=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') >> $setup_log 2>&1 + [[ $airgap_mngr_str == "True" ]] && is_airgap=true >> $setup_log 2>&1 +fi + +reset_proxy +if [[ -z $is_airgap ]]; then + collect_net_method + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 +fi + +if [[ $is_minion ]]; then + whiptail_ssh_key_copy_notice + copy_ssh_key >> $setup_log 2>&1 +fi + percentage=0 { installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs From eee612e73da33b9028a0e34e21cc7323c3970fca Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 27 Oct 2021 11:43:09 -0400 Subject: [PATCH 067/110] Make folder/file states explicit Rather than using /nsm/zeek (max_depth: 1) create explicit states for /nsm/zeek/spool and /nsm/zeek/spool/state.db that set correct ownership --- salt/zeek/init.sls | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 5f5adfaa6..f83293742 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -90,11 +90,14 @@ zeekpolicysync: # Ensure the zeek spool tree (and state.db) ownership is correct zeekspoolownership: file.directory: - - name: /nsm/zeek + - name: /nsm/zeek/spool - user: 937 - - max_depth: 1 - - recurse: - - user +zeekstatedbownership: + file.managed: + - name: /nsm/zeek/spool/state.db + - user: 937 + - replace: False + - create: False # Sync Intel zeekintelloadsync: @@ -247,4 +250,4 @@ delete_so-zeek_so-status.disabled: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From 75490a253639c9685dc8cc85f67a3e7e94ce4d2f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 27 Oct 2021 14:59:24 -0400 Subject: [PATCH 068/110] Fix typo --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 982ce5f05..58059567c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -326,7 +326,7 @@ if [[ ( $is_manager || $is_import) && $is_iso ]]; then if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true fi -el [[ $is_minion ]] +elif [[ $is_minion ]]; then airgap_mngr_str=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') >> $setup_log 2>&1 [[ $airgap_mngr_str == "True" ]] && is_airgap=true >> $setup_log 2>&1 fi From c8c8cf203f54fdc42051bf1864d2de13907e9d29 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 27 Oct 2021 15:44:52 -0400 Subject: [PATCH 069/110] Enable cluster stats --- salt/telegraf/etc/telegraf.conf | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 59d806fe0..649fcc241 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -16,9 +16,9 @@ {%- set MANAGER = salt['grains.get']('master') %} {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} -{% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} -{% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} +{%- set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} +{%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} {%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} # Global tags can be specified here in key="value" format. @@ -621,23 +621,25 @@ # # Read stats from one or more Elasticsearch servers or clusters -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} +{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] servers = ["https://{{ MANAGER }}:9200"] -{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + cluster_stats = true +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} username = "{{ ES_USER }}" password = "{{ ES_PASS }}" -{% endif %} +{%- endif %} insecure_skip_verify = true -{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} +{%- elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] -{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + cluster_stats = true +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} username = "{{ ES_USER }}" password = "{{ ES_PASS }}" -{% endif %} +{%- endif %} insecure_skip_verify = true -{% endif %} +{%- endif %} # # ## Timeout for HTTP requests to the elastic search server(s) From 18ce9c781943a23f371a722fee96877311518ba8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 28 Oct 2021 07:46:02 -0400 Subject: [PATCH 070/110] disable zeekpacketlosscron and telegraf checks if zeek is diabled via pillar --- salt/telegraf/etc/telegraf.conf | 23 +++++++++++++---------- salt/zeek/init.sls | 2 +- salt/zeek/map.jinja | 4 +++- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 649fcc241..92fcb4106 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -20,6 +20,9 @@ {%- set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} {%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} +{%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', 'True') %} +{%- set MDENGINE = salt['pillar.get']('global:mdengine', 'ZEEK') %} + # Global tags can be specified here in key="value" format. [global_tags] @@ -740,10 +743,10 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'ZEEK' %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", - {% endif %} + {%- endif %} "/scripts/oldpcap.sh", "/scripts/raid.sh", "/scripts/beatseps.sh" @@ -757,10 +760,10 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'ZEEK' %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", - {% endif %} + {%- endif %} "/scripts/oldpcap.sh", "/scripts/eps.sh", "/scripts/raid.sh", @@ -776,10 +779,10 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'ZEEK' %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", - {% endif %} + {%- endif %} "/scripts/oldpcap.sh", "/scripts/eps.sh", "/scripts/raid.sh", @@ -794,10 +797,10 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'ZEEK' %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", - {% endif %} + {%- endif %} "/scripts/oldpcap.sh", "/scripts/influxdbsize.sh", "/scripts/raid.sh", @@ -811,10 +814,10 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'ZEEK' %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", - {% endif %} + {%- endif %} "/scripts/oldpcap.sh", "/scripts/helixeps.sh" ] diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index e4b83a9e1..2c9fb9846 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -146,7 +146,7 @@ plcronscript: - mode: 755 zeekpacketlosscron: - cron.present: + cron.{{ZEEKOPTIONS.pl_cron_state}}: - name: /usr/local/bin/packetloss.sh - user: root - minute: '*/10' diff --git a/salt/zeek/map.jinja b/salt/zeek/map.jinja index b5713c6d5..5ae8894bc 100644 --- a/salt/zeek/map.jinja +++ b/salt/zeek/map.jinja @@ -4,12 +4,14 @@ # don't start the docker container if it is an import node or disabled via pillar {% if grains.id.split('_')|last == 'import' or ENABLED is sameas false %} {% do ZEEKOPTIONS.update({'start': False}) %} + {% do ZEEKOPTIONS.update({'pl_cron_state': 'absent'}) %} {% else %} {% do ZEEKOPTIONS.update({'start': True}) %} + {% do ZEEKOPTIONS.update({'pl_cron_state': 'present'}) %} {% endif %} {% if ENABLED is sameas false %} {% do ZEEKOPTIONS.update({'status': 'absent'}) %} {% else %} {% do ZEEKOPTIONS.update({'status': 'running'}) %} -{% endif %} \ No newline at end of file +{% endif %} From 7eb42fa6bdb386562a03970fb94bb20ed316f822 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 28 Oct 2021 08:43:03 -0400 Subject: [PATCH 071/110] change boolean --- salt/telegraf/etc/telegraf.conf | 12 ++++++------ salt/zeek/map.jinja | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 92fcb4106..fa7dabe09 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -20,7 +20,7 @@ {%- set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} {%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} -{%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', 'True') %} +{%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %} {%- set MDENGINE = salt['pillar.get']('global:mdengine', 'ZEEK') %} @@ -743,7 +743,7 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", {%- endif %} @@ -760,7 +760,7 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", {%- endif %} @@ -779,7 +779,7 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", {%- endif %} @@ -797,7 +797,7 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", {%- endif %} @@ -814,7 +814,7 @@ "/scripts/stenoloss.sh", "/scripts/suriloss.sh", "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED is sameas true %} + {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} "/scripts/zeekloss.sh", "/scripts/zeekcaptureloss.sh", {%- endif %} diff --git a/salt/zeek/map.jinja b/salt/zeek/map.jinja index 5ae8894bc..a0f92463e 100644 --- a/salt/zeek/map.jinja +++ b/salt/zeek/map.jinja @@ -1,8 +1,8 @@ {% set ZEEKOPTIONS = {} %} -{% set ENABLED = salt['pillar.get']('zeek:enabled', 'True') %} +{% set ENABLED = salt['pillar.get']('zeek:enabled', True) %} # don't start the docker container if it is an import node or disabled via pillar -{% if grains.id.split('_')|last == 'import' or ENABLED is sameas false %} +{% if grains.id.split('_')|last == 'import' or not ENABLED %} {% do ZEEKOPTIONS.update({'start': False}) %} {% do ZEEKOPTIONS.update({'pl_cron_state': 'absent'}) %} {% else %} @@ -10,7 +10,7 @@ {% do ZEEKOPTIONS.update({'pl_cron_state': 'present'}) %} {% endif %} -{% if ENABLED is sameas false %} +{% if not ENABLED %} {% do ZEEKOPTIONS.update({'status': 'absent'}) %} {% else %} {% do ZEEKOPTIONS.update({'status': 'running'}) %} From 0cea5e8f2269ca6147fbf83ef015c5bb34ac2b2a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 28 Oct 2021 08:46:27 -0400 Subject: [PATCH 072/110] include ssl state in telegraf state --- salt/telegraf/init.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index d71916430..2c188367b 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -5,6 +5,9 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +include: + - ssl + # Add Telegraf to monitor all the things. tgraflogdir: file.directory: From d6eeb0b735aae8ef6c642e107893e038d65d2b34 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 28 Oct 2021 10:04:03 -0400 Subject: [PATCH 073/110] Gen ssh key sooner --- setup/so-setup | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 58059567c..e9d510270 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -304,6 +304,11 @@ if ! [[ -f $install_opt_file ]]; then add_mngr_ip_to_hosts fi + if [[ $is_minion ]]; then + whiptail_ssh_key_copy_notice + copy_ssh_key >> $setup_log 2>&1 + fi + if [[ $is_minion ]] && ! (compare_versions); then info "Installer version mismatch, downloading correct version from manager" printf '%s\n' \ @@ -337,11 +342,6 @@ if [[ -z $is_airgap ]]; then [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi -if [[ $is_minion ]]; then - whiptail_ssh_key_copy_notice - copy_ssh_key >> $setup_log 2>&1 -fi - percentage=0 { installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs From 6b480a5ba448569c7a024dbbb18afd13dab16441 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 28 Oct 2021 11:51:50 -0400 Subject: [PATCH 074/110] Change airgap check to something that doesn't require root --- setup/so-setup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index e9d510270..b423160c8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -332,8 +332,9 @@ if [[ ( $is_manager || $is_import) && $is_iso ]]; then is_airgap=true fi elif [[ $is_minion ]]; then - airgap_mngr_str=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') >> $setup_log 2>&1 - [[ $airgap_mngr_str == "True" ]] && is_airgap=true >> $setup_log 2>&1 + $sshcmd -i /root/.ssh/so.key soremote@"$MSRV" [[ -f /etc/yum.repos.d/airgap_repo.repo ]] >> $setup_log 2>&1 + airgap_check=$? + [[ $airgap_check ]] && is_airgap=true >> $setup_log 2>&1 fi reset_proxy From c2b18efdbbd1658b3a3ec1a1da77eafc32444cf6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 28 Oct 2021 11:59:42 -0400 Subject: [PATCH 075/110] Minions still need to be ISO installs to be airgap --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index b423160c8..a2ffdf339 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -331,7 +331,7 @@ if [[ ( $is_manager || $is_import) && $is_iso ]]; then if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true fi -elif [[ $is_minion ]]; then +elif [[ $is_minion && $is_iso ]]; then $sshcmd -i /root/.ssh/so.key soremote@"$MSRV" [[ -f /etc/yum.repos.d/airgap_repo.repo ]] >> $setup_log 2>&1 airgap_check=$? [[ $airgap_check ]] && is_airgap=true >> $setup_log 2>&1 From f081938be5195c741f577e42eaa68e59ffadf5e6 Mon Sep 17 00:00:00 2001 From: Burak-PLT <55863083+Burak-PLT@users.noreply.github.com> Date: Thu, 28 Oct 2021 16:00:58 -0400 Subject: [PATCH 076/110] Update auth.sls Change default password lengths to 72 characters from 20. --- salt/elasticsearch/auth.sls | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index 373f2fbed..35d669892 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -1,8 +1,8 @@ -{% set so_elastic_user_pass = salt['random.get_str'](20) %} -{% set so_kibana_user_pass = salt['random.get_str'](20) %} -{% set so_logstash_user_pass = salt['random.get_str'](20) %} -{% set so_beats_user_pass = salt['random.get_str'](20) %} -{% set so_monitor_user_pass = salt['random.get_str'](20) %} +{% set so_elastic_user_pass = salt['random.get_str'](72) %} +{% set so_kibana_user_pass = salt['random.get_str'](72) %} +{% set so_logstash_user_pass = salt['random.get_str'](72) %} +{% set so_beats_user_pass = salt['random.get_str'](72) %} +{% set so_monitor_user_pass = salt['random.get_str'](72) %} elastic_auth_pillar: file.managed: From d08149f728b27dcff5b1833465922ab9667b6629 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 29 Oct 2021 10:11:47 -0400 Subject: [PATCH 077/110] Don't set `INTERWEBS` variable on automated minions --- setup/automation/distributed-airgap-search | 1 - setup/automation/distributed-airgap-sensor | 1 - 2 files changed, 2 deletions(-) diff --git a/setup/automation/distributed-airgap-search b/setup/automation/distributed-airgap-search index 1acee9b1a..7a0888fee 100644 --- a/setup/automation/distributed-airgap-search +++ b/setup/automation/distributed-airgap-search @@ -35,7 +35,6 @@ ADMINPASS2=onionuser HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit HOSTNAME=distributed-search -INTERWEBS=AIRGAP install_type=SEARCHNODE # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= diff --git a/setup/automation/distributed-airgap-sensor b/setup/automation/distributed-airgap-sensor index c8186bf8a..91b9c24a9 100644 --- a/setup/automation/distributed-airgap-sensor +++ b/setup/automation/distributed-airgap-sensor @@ -35,7 +35,6 @@ ZEEKVERSION=ZEEK HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit HOSTNAME=distributed-sensor -INTERWEBS=AIRGAP install_type=SENSOR # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= From b1c67f696e9a261e85e4c079cd4af3330931c0a1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 29 Oct 2021 10:47:05 -0400 Subject: [PATCH 078/110] Re-order logic to maintain backwards compatibility --- setup/so-setup | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index a2ffdf339..2cef6d876 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -309,6 +309,24 @@ if ! [[ -f $install_opt_file ]]; then copy_ssh_key >> $setup_log 2>&1 fi + # Check if this is an airgap install + if [[ ( $is_manager || $is_import) && $is_iso ]]; then + whiptail_airgap + if [[ "$INTERWEBS" == 'AIRGAP' ]]; then + is_airgap=true + fi + elif [[ $is_minion && $is_iso ]]; then + $sshcmd -i /root/.ssh/so.key soremote@"$MSRV" [[ -f /etc/yum.repos.d/airgap_repo.repo ]] >> $setup_log 2>&1 + airgap_check=$? + [[ $airgap_check ]] && is_airgap=true >> $setup_log 2>&1 + fi + + reset_proxy + if [[ -z $is_airgap ]]; then + collect_net_method + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 + fi + if [[ $is_minion ]] && ! (compare_versions); then info "Installer version mismatch, downloading correct version from manager" printf '%s\n' \ @@ -317,7 +335,9 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" \ "MSRV=$MSRV" \ "MSRVIP=$MSRVIP" \ + "is_airgap=$is_airgap" \ "NODE_DESCRIPTION=\"$NODE_DESCRIPTION\"" > "$install_opt_file" + [[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi @@ -325,24 +345,6 @@ else rm -rf $install_opt_file >> "$setup_log" 2>&1 fi -# Check if this is an airgap install -if [[ ( $is_manager || $is_import) && $is_iso ]]; then - whiptail_airgap - if [[ "$INTERWEBS" == 'AIRGAP' ]]; then - is_airgap=true - fi -elif [[ $is_minion && $is_iso ]]; then - $sshcmd -i /root/.ssh/so.key soremote@"$MSRV" [[ -f /etc/yum.repos.d/airgap_repo.repo ]] >> $setup_log 2>&1 - airgap_check=$? - [[ $airgap_check ]] && is_airgap=true >> $setup_log 2>&1 -fi - -reset_proxy -if [[ -z $is_airgap ]]; then - collect_net_method - [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 -fi - percentage=0 { installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs From f76a52b2ee1dc0cb26bed76b6564372abe70ed6b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 29 Oct 2021 13:34:23 -0400 Subject: [PATCH 079/110] Fix NIC string values for VLAN tagged interfaces --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 62d458911..37e9b5ba9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -846,7 +846,7 @@ check_requirements() { local req_cores local req_storage local nic_list - readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0" | sed 's/ //g')" + readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g')" local num_nics=${#nic_list[@]} if [[ "$standalone_or_dist" == 'standalone' ]]; then @@ -1374,7 +1374,7 @@ filter_unused_nics() { fi # Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use) - filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string" | sed 's/ //g') + filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g') readarray -t filtered_nics <<< "$filtered_nics" nic_list=() From 2ba619144cc6a87a15a3d17e87e68337e0206748 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Nov 2021 08:23:29 -0400 Subject: [PATCH 080/110] Support non-WEL Beats --- salt/elasticsearch/files/ingest/beats.common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/beats.common b/salt/elasticsearch/files/ingest/beats.common index 4e358582e..3cfa33521 100644 --- a/salt/elasticsearch/files/ingest/beats.common +++ b/salt/elasticsearch/files/ingest/beats.common @@ -2,7 +2,7 @@ "description" : "beats.common", "processors" : [ { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, - { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, + { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } }, { "pipeline": { "name": "common" } } ] } \ No newline at end of file From b756c0cd38244f3eda728658710504eb6f54650c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Nov 2021 08:57:11 -0400 Subject: [PATCH 081/110] Pull ES Creds at Runtime --- salt/common/tools/sbin/so-import-evtx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx index 9e640beaa..274a2835d 100755 --- a/salt/common/tools/sbin/so-import-evtx +++ b/salt/common/tools/sbin/so-import-evtx @@ -41,13 +41,16 @@ function evtx2es() { EVTX=$1 HASH=$2 + ES_PW=$(lookup_pillar "auth:users:so_elastic_user:pass" "elasticsearch") + ES_USER=$(lookup_pillar "auth:users:so_elastic_user:user" "elasticsearch") + docker run --rm \ -v "$EVTX:/tmp/$RUNID.evtx" \ --entrypoint evtx2es \ {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} \ --host {{ MANAGERIP }} --scheme https \ --index so-beats-$INDEX_DATE --pipeline import.wel \ - --login {{ES_USER}} --pwd {{ES_PW}} \ + --login $ES_USER --pwd $ES_PW \ "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null docker run --rm \ From 35342565170b756e74e449c856b3061dea6f6174 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Nov 2021 09:03:52 -0400 Subject: [PATCH 082/110] Add evtx import logging --- salt/common/tools/sbin/so-import-evtx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx index 274a2835d..d00e4b13d 100755 --- a/salt/common/tools/sbin/so-import-evtx +++ b/salt/common/tools/sbin/so-import-evtx @@ -25,6 +25,7 @@ INDEX_DATE=$(date +'%Y.%m.%d') RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1) +LOG_FILE=/nsm/import/evtx-import.log . /usr/sbin/so-common @@ -51,7 +52,7 @@ function evtx2es() { --host {{ MANAGERIP }} --scheme https \ --index so-beats-$INDEX_DATE --pipeline import.wel \ --login $ES_USER --pwd $ES_PW \ - "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null + "/tmp/$RUNID.evtx" 1>/dev/null > $LOG_FILE 2>&1 docker run --rm \ -v "$EVTX:/tmp/import.evtx" \ From a6f399acf45f5706bf4e147057846ede4e1a0374 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Nov 2021 09:19:32 -0400 Subject: [PATCH 083/110] Fix evtx import logging --- salt/common/tools/sbin/so-import-evtx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx index d00e4b13d..83815eecd 100755 --- a/salt/common/tools/sbin/so-import-evtx +++ b/salt/common/tools/sbin/so-import-evtx @@ -52,7 +52,7 @@ function evtx2es() { --host {{ MANAGERIP }} --scheme https \ --index so-beats-$INDEX_DATE --pipeline import.wel \ --login $ES_USER --pwd $ES_PW \ - "/tmp/$RUNID.evtx" 1>/dev/null > $LOG_FILE 2>&1 + "/tmp/$RUNID.evtx" >> $LOG_FILE 2>&1 docker run --rm \ -v "$EVTX:/tmp/import.evtx" \ From 9a9d1480de89eab71cd2ba2f9a21073955dd6741 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Nov 2021 10:41:36 -0400 Subject: [PATCH 084/110] Manage docker group's gid to prevent gid overlap --- salt/common/init.sls | 5 +++++ salt/salt/minion.sls | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 05dd7023f..17cea3480 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -9,6 +9,11 @@ rmvariablesfile: file.absent: - name: /tmp/variables.txt +dockergroup: + group.present: + - name: docker + - gid: 920 + # Add socore Group socoregroup: group.present: diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 3db257d1b..04fc1769c 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -87,4 +87,4 @@ salt_minion_service: patch_pkg: pkg.installed: - - name: patch \ No newline at end of file + - name: patch From e6adb46364db153c66b51a4bde00bb172e8a5de1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Nov 2021 11:18:23 -0400 Subject: [PATCH 085/110] Run so-preflight during setup --- setup/so-functions | 3 ++- setup/so-preflight | 40 +++++++++++++++++++++++++++++++++------- setup/so-setup | 17 ++++++++++++++++- setup/so-whiptail | 16 +++++++++++++++- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 37e9b5ba9..6d46b4bb4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1124,9 +1124,10 @@ detect_os() { installer_progress_loop() { local i=0 + local msg="${1:-Performing background actions...}" while true; do [[ $i -lt 98 ]] && ((i++)) - set_progress_str "$i" 'Checking that all required packages are installed and enabled...' nolog + set_progress_str "$i" "$msg" nolog [[ $i -gt 0 ]] && sleep 5s done } diff --git a/setup/so-preflight b/setup/so-preflight index 2943191eb..e30b7872d 100644 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -18,7 +18,13 @@ source ../salt/common/tools/sbin/so-common source ./so-functions -preflight_log='/root/preflight.log' +script_run="$1" + +if [[ $script_run == true ]]; then + preflight_log="${2:-'/root/preflight.log'}" +else + preflight_log='/root/preflight.log' +fi check_default_repos() { local ret_code=0 @@ -72,7 +78,8 @@ check_new_repos() { check_misc_urls() { printf ' Checking various other URLs used by setup.' | tee -a "$preflight_log" - local so_version=$(cat ../VERSION) + local so_version + so_version=$(cat ../VERSION) local url_arr=( "https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" "https://github.com/Neo23x0/signature-base" @@ -112,10 +119,18 @@ __check_url_arr() { } main() { + local intro_str="Beginning pre-flight checks." + local success_str="Pre-flight checks completed successfully!" + local fail_str="Pre-flight checks could not complete." + detect_os "$preflight_log" [[ -f $preflight_log ]] || touch "$preflight_log" - echo "Beginning pre-flight checks." | tee "$preflight_log" + if [[ $script_run == true ]]; then + echo "$intro_str" + else + echo "$intro_str" | tee "$preflight_log" + fi check_default_repos &&\ check_new_repos &&\ check_misc_urls @@ -124,12 +139,23 @@ main() { echo "" if [[ $success == 0 ]]; then - echo -e "Pre-flight checks completed successfully!\n" | tee -a "$preflight_log" + if [[ $script_run == true ]]; then + echo "$success_str" + else + echo "$success_str" | tee -a "$preflight_log" + echo "" + fi else - echo -e "Pre-flight checks could not complete." | tee -a "$preflight_log" - echo -e " Check $preflight_log for details.\n" - exit 1 + if [[ $script_run == true ]]; then + echo "$fail_str" + else + echo "$fail_str" | tee -a "$preflight_log" + echo "Check $preflight_log for details." + echo "" + fi fi + + exit $success } main diff --git a/setup/so-setup b/setup/so-setup index 2cef6d876..2dbf9935e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -345,9 +345,24 @@ else rm -rf $install_opt_file >> "$setup_log" 2>&1 fi +if [[ -z $is_airgap ]]; then + percentage=0 + { + installer_progress_loop 'Running preflight checks...' & + progress_bg_proc=$! + ./so-preflight true "$setup_log" >> $setup_log 2>&1 + preflight_ret=$? + kill -9 "$progress_bg_proc" + wait "$progress_bg_proc" &> /dev/null + if [[ $preflight_ret -gt 0 ]] && ! ( whiptail_preflight_err ); then + whiptail_cancel + fi + } | progress '...' +fi + percentage=0 { - installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs + installer_progress_loop 'Checking that all required packages are installed and enabled...' & # Run progress bar to 98 in ~8 minutes while waiting for package installs progress_bg_proc=$! installer_prereq_packages install_success=$? diff --git a/setup/so-whiptail b/setup/so-whiptail index 130ae96bb..371897eb8 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -95,7 +95,7 @@ whiptail_cancel() { title "User cancelled setup." - exit + exit 1 } whiptail_check_exitstatus() { @@ -1489,6 +1489,20 @@ whiptail_patch_schedule_select_hours() { } +whiptail_preflight_err() { + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + The so-preflight script failed checking one or more URLs required by setup. Check $setup_log for more details. + + Would you like to exit setup? + EOM + + whiptail --title "$whiptail_title" \ + --yesno "$message" 11 75 \ + --yes-button "Continue" --no-button "Exit" --defaultno +} + whiptail_proxy_ask() { [ -n "$TESTING" ] && return From 9671dab2a3a5d9bf7ba21ed7b9d2efd90772c16e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Nov 2021 11:48:24 -0400 Subject: [PATCH 086/110] Make so-preflight executable --- setup/so-preflight | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup/so-preflight diff --git a/setup/so-preflight b/setup/so-preflight old mode 100644 new mode 100755 From e4a77acfe6b490ef43c0d0924b7b1199e7db7914 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Nov 2021 12:03:42 -0400 Subject: [PATCH 087/110] Move whiptail menus outside of progress func --- setup/so-setup | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 2dbf9935e..b4e469ced 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -352,12 +352,15 @@ if [[ -z $is_airgap ]]; then progress_bg_proc=$! ./so-preflight true "$setup_log" >> $setup_log 2>&1 preflight_ret=$? + echo "$preflight_ret" > /tmp/preflight_ret kill -9 "$progress_bg_proc" wait "$progress_bg_proc" &> /dev/null - if [[ $preflight_ret -gt 0 ]] && ! ( whiptail_preflight_err ); then - whiptail_cancel - fi } | progress '...' + [[ -f /tmp/setup_tmp_var ]] && preflight_ret=$(cat /tmp/preflight_ret) + rm /tmp/preflight_ret + if [[ -n $preflight_ret && $preflight_ret -gt 0 ]] && ! ( whiptail_preflight_err ); then + whiptail_cancel + fi fi percentage=0 From 7c7c225a415695fd854643e674e369c43e61f673 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Nov 2021 14:01:21 -0400 Subject: [PATCH 088/110] Fix tmp file check --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index b4e469ced..d71511971 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -356,7 +356,7 @@ if [[ -z $is_airgap ]]; then kill -9 "$progress_bg_proc" wait "$progress_bg_proc" &> /dev/null } | progress '...' - [[ -f /tmp/setup_tmp_var ]] && preflight_ret=$(cat /tmp/preflight_ret) + [[ -f /tmp/preflight_ret ]] && preflight_ret=$(cat /tmp/preflight_ret) rm /tmp/preflight_ret if [[ -n $preflight_ret && $preflight_ret -gt 0 ]] && ! ( whiptail_preflight_err ); then whiptail_cancel From 8670aa6cd84e4b8d5ab8118567228d289eb6a492 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Nov 2021 14:29:58 -0400 Subject: [PATCH 089/110] Run check-update in preflight instead of update --- setup/so-preflight | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-preflight b/setup/so-preflight index e30b7872d..fd6b5f736 100755 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -33,7 +33,7 @@ check_default_repos() { if [[ $OS == 'centos' ]]; then printf '%s' 'yum update.' | tee -a "$preflight_log" echo "" >> "$preflight_log" - yum -y update >> $preflight_log 2>&1 + yum -y check-update >> $preflight_log 2>&1 ret_code=$? else printf '%s' 'apt update.' | tee -a "$preflight_log" From d927e79154ce318cad8513385d56077ff1c5b7b2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Nov 2021 16:17:08 -0400 Subject: [PATCH 090/110] Exit on failed preflight check during testing --- setup/so-whiptail | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 371897eb8..cbfa5a886 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -84,6 +84,8 @@ whiptail_bond_nics_mtu() { whiptail_cancel() { + [ -n "$TESTING" ] && exit 1 + whiptail --title "$whiptail_title" --msgbox "Cancelling Setup." 8 75 if [ -d "/root/installtmp" ]; then { @@ -1490,7 +1492,7 @@ whiptail_patch_schedule_select_hours() { } whiptail_preflight_err() { - [ -n "$TESTING" ] && return + [ -n "$TESTING" ] && return 1 read -r -d '' message <<- EOM The so-preflight script failed checking one or more URLs required by setup. Check $setup_log for more details. From fb35ff40b49451b1ce3e27cd296b2e4f2e3eb971 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Nov 2021 09:19:41 -0400 Subject: [PATCH 091/110] Just hide whiptail cancel message on test installs --- setup/so-whiptail | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index cbfa5a886..95650415c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -83,10 +83,8 @@ whiptail_bond_nics_mtu() { } whiptail_cancel() { - - [ -n "$TESTING" ] && exit 1 - - whiptail --title "$whiptail_title" --msgbox "Cancelling Setup." 8 75 + [ -z "$TESTING" ] && whiptail --title "$whiptail_title" --msgbox "Cancelling Setup." 8 75 + if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; From 747f14d60e558059468665b01126fcc20d996e18 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Nov 2021 13:11:38 -0400 Subject: [PATCH 092/110] Make common template honor replicas --- .../templates/so/so-common-template.json.jinja | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json.jinja b/salt/elasticsearch/templates/so/so-common-template.json.jinja index 4394ebb65..3ffae5c84 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-common-template.json.jinja @@ -1,12 +1,14 @@ {%- set INDEX_SORTING = salt['pillar.get']('elasticsearch:index_sorting', True) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cisco:refresh', '30s') %} { "index_patterns": ["so-*"], "version":50001, "order":10, "settings":{ - "number_of_replicas":0, + "number_of_replicas":{{ REPLICAS }}, "number_of_shards":1, - "index.refresh_interval":"30s", + "index.refresh_interval":"{{ REFRESH }}", "index.routing.allocation.require.box_type":"hot", "index.mapping.total_fields.limit": "1500", {%- if INDEX_SORTING is sameas true %} From dc07aba63dd3c73f91740aacc07257f0a75a97d2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Nov 2021 13:50:31 -0400 Subject: [PATCH 093/110] Update so-common-template.json.jinja --- salt/elasticsearch/templates/so/so-common-template.json.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json.jinja b/salt/elasticsearch/templates/so/so-common-template.json.jinja index 3ffae5c84..4a41cba8a 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-common-template.json.jinja @@ -1,6 +1,6 @@ {%- set INDEX_SORTING = salt['pillar.get']('elasticsearch:index_sorting', True) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cisco:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-common:refresh', '30s') %} { "index_patterns": ["so-*"], "version":50001, From 988932293f41f5cad5571b1dfd3f70be6d187cee Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Nov 2021 15:54:17 -0400 Subject: [PATCH 094/110] Whiptail changes * Ask whether to join to or create new dist install * Also add links to architecture on install type prompts --- setup/so-whiptail | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 95650415c..8e137b8f5 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -735,7 +735,7 @@ whiptail_install_type() { # What kind of install are we doing? install_type=$(whiptail --title "$whiptail_title" --radiolist \ - "Choose install type:" 12 65 5 \ + "Choose install type. See https://docs.securityonion.net/architecture for details." 12 65 5 \ "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ "DISTRIBUTED" "Distributed install submenu " OFF \ @@ -749,6 +749,11 @@ whiptail_install_type() { if [[ $install_type == "DISTRIBUTED" ]]; then whiptail_install_type_dist + if [[ $dist_option == "NEWDEPLOYMENT" ]]; then + whiptail_install_type_dist_new + else + whiptail_install_type_dist_existing + fi elif [[ $install_type == "OTHER" ]]; then whiptail_install_type_other fi @@ -759,13 +764,39 @@ whiptail_install_type() { whiptail_install_type_dist() { [ -n "$TESTING" ] && return + + dist_option=$(whiptail --title "$whiptail_title" --menu "Do you want to start a new deployment or join this box to an existing deployment?" 10 75 2 \ + "New Deployment " "Create a new Security Onion deployment" \ + "Existing Deployment " "Join to an exisiting Security Onion deployment " \ + 3>&1 1>&2 2>&3 + ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + dist_option=$(echo "${option^^}" | tr -d ' ') +} + +whiptail_install_type_dist_new() { + [ -n "$TESTING" ] && return install_type=$(whiptail --title "$whiptail_title" --radiolist \ - "Choose distributed node type:" 13 60 6 \ - "MANAGER" "Start a new grid " ON \ + "Choose distributed manager type. See https://docs.securityonion.net/architecture for details." 24 60 6 \ + "MANAGER" "Start a new grid - requires separate search node(s) " ON \ + "MANAGERSEARCH" "Start a new grid - separate search node(s) are optional " OFF \ + 3>&1 1>&2 2>&3 + ) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + +whiptail_install_type_dist_existing() { + [ -n "$TESTING" ] && return + + install_type=$(whiptail --title "$whiptail_title" --radiolist \ + "Choose distributed node type. See https://docs.securityonion.net/architecture for details." 13 60 6 \ "SENSOR" "Create a forward only sensor " OFF \ "SEARCHNODE" "Add a search node with parsing " OFF \ - "MANAGERSEARCH" "Manager + search node " OFF \ "FLEET" "Dedicated Fleet Osquery Node " OFF \ "HEAVYNODE" "Sensor + Search Node " OFF \ 3>&1 1>&2 2>&3 @@ -777,8 +808,6 @@ whiptail_install_type_dist() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - - export install_type } whiptail_install_type_other() { @@ -907,6 +936,7 @@ whiptail_first_menu_iso() { option=$(echo "${option^^}" | tr -d ' ') } + whiptail_make_changes() { [ -n "$TESTING" ] && return From 246d41c55270292454f7121914cbcbdeb6f904d1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Nov 2021 15:56:08 -0400 Subject: [PATCH 095/110] Add additional checks for manager hostname + ip Check for current hostname, ip, and localhost (ip + string) when setting the manager ip and hostname --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 6d46b4bb4..410cc5970 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -470,7 +470,7 @@ collect_int_ip_mask() { collect_mngr_hostname() { whiptail_management_server - while ! valid_hostname "$MSRV"; do + while ! valid_hostname "$MSRV" || [[ $MSRV == "$HOSTNAME" || $MSRVIP == "localhost" ]]; do whiptail_invalid_hostname whiptail_management_server "$MSRV" done @@ -478,7 +478,7 @@ collect_mngr_hostname() { if ! getent hosts "$MSRV"; then whiptail_manager_ip - while ! valid_ip4 "$MSRVIP"; do + while ! valid_ip4 "$MSRVIP" || [[ $MSRVIP == "$MAINIP" || $MSRVIP == "127.0.0.1" ]]; do whiptail_invalid_input whiptail_manager_ip "$MSRVIP" done From dcf6dfb676f7572c176983b517072b9d3a69860b Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 8 Nov 2021 06:38:16 -0500 Subject: [PATCH 096/110] Improve clarity --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02ab437fb..aeb33ad8f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ ### Contributing code -* **All commits must be signed** with a valid key that has been added to your GitHub account. The commits should have all the "**Verified**" tag when viewed on GitHub as shown below: +* **All commits must be signed** with a valid key that has been added to your GitHub account. Each commit should have the "**Verified**" tag when viewed on GitHub as shown below: From 9c4bba9ac95a0d039967eae4568fd5248a32ccfb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 10:08:23 -0500 Subject: [PATCH 097/110] Fix variable reference --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 8e137b8f5..08a1d369e 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -773,7 +773,7 @@ whiptail_install_type_dist() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - dist_option=$(echo "${option^^}" | tr -d ' ') + dist_option=$(echo "${dist_option^^}" | tr -d ' ') } whiptail_install_type_dist_new() { From dbe4a7de6342ccce451454446808c0e82449d542 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 10:19:38 -0500 Subject: [PATCH 098/110] Fix new whiptail layouts --- setup/so-whiptail | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 08a1d369e..8d7782272 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -780,9 +780,9 @@ whiptail_install_type_dist_new() { [ -n "$TESTING" ] && return install_type=$(whiptail --title "$whiptail_title" --radiolist \ - "Choose distributed manager type. See https://docs.securityonion.net/architecture for details." 24 60 6 \ - "MANAGER" "Start a new grid - requires separate search node(s) " ON \ - "MANAGERSEARCH" "Start a new grid - separate search node(s) are optional " OFF \ + "Choose distributed manager type to start a new grid. See https://docs.securityonion.net/architecture for details." 10 75 2 \ + "MANAGER" "New grid, requires separate search node(s) " ON \ + "MANAGERSEARCH" "New grid, separate search node(s) are optional " OFF \ 3>&1 1>&2 2>&3 ) @@ -794,8 +794,8 @@ whiptail_install_type_dist_existing() { [ -n "$TESTING" ] && return install_type=$(whiptail --title "$whiptail_title" --radiolist \ - "Choose distributed node type. See https://docs.securityonion.net/architecture for details." 13 60 6 \ - "SENSOR" "Create a forward only sensor " OFF \ + "Choose distributed node type to join to an existing grid. See https://docs.securityonion.net/architecture for details." 14 57 4 \ + "SENSOR" "Create a forward only sensor " ON \ "SEARCHNODE" "Add a search node with parsing " OFF \ "FLEET" "Dedicated Fleet Osquery Node " OFF \ "HEAVYNODE" "Sensor + Search Node " OFF \ From 8b2cccdf4ab367319010e29fa1b70c8e2e4e13ac Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 10:21:17 -0500 Subject: [PATCH 099/110] More whiptail formatting --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 8d7782272..2f3b54215 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -765,7 +765,7 @@ whiptail_install_type_dist() { [ -n "$TESTING" ] && return - dist_option=$(whiptail --title "$whiptail_title" --menu "Do you want to start a new deployment or join this box to an existing deployment?" 10 75 2 \ + dist_option=$(whiptail --title "$whiptail_title" --menu "Do you want to start a new deployment or join this box to \nan existing deployment?" 10 75 2 \ "New Deployment " "Create a new Security Onion deployment" \ "Existing Deployment " "Join to an exisiting Security Onion deployment " \ 3>&1 1>&2 2>&3 From ad71485361d13accc6fd70fc70756ed685a0e740 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 10:21:55 -0500 Subject: [PATCH 100/110] Fix whiptail height --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 2f3b54215..877daf966 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -765,7 +765,7 @@ whiptail_install_type_dist() { [ -n "$TESTING" ] && return - dist_option=$(whiptail --title "$whiptail_title" --menu "Do you want to start a new deployment or join this box to \nan existing deployment?" 10 75 2 \ + dist_option=$(whiptail --title "$whiptail_title" --menu "Do you want to start a new deployment or join this box to \nan existing deployment?" 11 75 2 \ "New Deployment " "Create a new Security Onion deployment" \ "Existing Deployment " "Join to an exisiting Security Onion deployment " \ 3>&1 1>&2 2>&3 From 50b7779d6e15d1608dafc665f2cb7a691031086e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 10:35:28 -0500 Subject: [PATCH 101/110] Make manager hostname error more specific --- setup/so-functions | 7 ++++++- setup/so-whiptail | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 410cc5970..0ac5df697 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -470,11 +470,16 @@ collect_int_ip_mask() { collect_mngr_hostname() { whiptail_management_server - while ! valid_hostname "$MSRV" || [[ $MSRV == "$HOSTNAME" || $MSRVIP == "localhost" ]]; do + while ! valid_hostname "$MSRV"; do whiptail_invalid_hostname whiptail_management_server "$MSRV" done + while [[ $MSRV == "$HOSTNAME" || $MSRVIP == "localhost" ]]; do + whiptail_invalid_hostname 0 + whiptail_management_server "$MSRV" + done + if ! getent hosts "$MSRV"; then whiptail_manager_ip diff --git a/setup/so-whiptail b/setup/so-whiptail index 877daf966..db5e36516 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -841,7 +841,6 @@ whiptail_invalid_input() { # TODO: This should accept a list of arguments to spe [ -n "$TESTING" ] && return whiptail --title "$whiptail_title" --msgbox " Invalid input, please try again." 7 40 - } whiptail_invalid_proxy() { @@ -888,10 +887,21 @@ whiptail_invalid_user_warning() { whiptail_invalid_hostname() { [ -n "$TESTING" ] && return + local is_manager_hostname + is_manager_hostname="$1" + local error_message - error_message=$(echo "Please choose a valid hostname. It cannot be localhost; and must contain only \ - the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', \ - and hyphen ('-')" | tr -d '\t') + read -r -d '' error_message <<- EOM + Please choose a valid hostname. It cannot be localhost; and must contain only the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', and hyphen ('-') + EOM + + if [[ $is_manager_hostname = 0 ]]; then + local error_message + read -r -d '' error_message <<- EOM + Please enter a valid hostname. The manager hostname cannot be localhost or the chosen hostname for this machine. + EOM + + fi whiptail --title "$whiptail_title" \ --msgbox "$error_message" 10 75 From ee2dd75dfd7c7394e3f9b5255e001b38d9bc3864 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 10:36:36 -0500 Subject: [PATCH 102/110] Fix variable ref --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 0ac5df697..2a8a6ff88 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -475,7 +475,7 @@ collect_mngr_hostname() { whiptail_management_server "$MSRV" done - while [[ $MSRV == "$HOSTNAME" || $MSRVIP == "localhost" ]]; do + while [[ $MSRV == "$HOSTNAME" || $MSRV == "localhost" ]]; do whiptail_invalid_hostname 0 whiptail_management_server "$MSRV" done From f66d915f5df06dc82976b971b14a9728bf1a6435 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 10:38:30 -0500 Subject: [PATCH 103/110] Normal hostname check already checks for localhost --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 2a8a6ff88..a9925c80d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -475,7 +475,7 @@ collect_mngr_hostname() { whiptail_management_server "$MSRV" done - while [[ $MSRV == "$HOSTNAME" || $MSRV == "localhost" ]]; do + while [[ $MSRV == "$HOSTNAME" ]]; do whiptail_invalid_hostname 0 whiptail_management_server "$MSRV" done From acba82d1948b298f3a9721abede8b7575440b90a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 11:04:51 -0500 Subject: [PATCH 104/110] Update dist install menus' top text --- setup/so-whiptail | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index db5e36516..3f8628a30 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -778,9 +778,17 @@ whiptail_install_type_dist() { whiptail_install_type_dist_new() { [ -n "$TESTING" ] && return + + local mngr_msg + read -r -d '' mngr_msg <<- EOM + Choose a distributed manager type to start a new grid. - install_type=$(whiptail --title "$whiptail_title" --radiolist \ - "Choose distributed manager type to start a new grid. See https://docs.securityonion.net/architecture for details." 10 75 2 \ + See https://docs.securityonion.net/architecture for details. + + Note: MANAGER is the recommended option for most users. MANAGERSEARCH should only be used in very specific situations. + EOM + + install_type=$(whiptail --title "$whiptail_title" --radiolist "$mngr_msg" 15 75 2 \ "MANAGER" "New grid, requires separate search node(s) " ON \ "MANAGERSEARCH" "New grid, separate search node(s) are optional " OFF \ 3>&1 1>&2 2>&3 @@ -792,9 +800,17 @@ whiptail_install_type_dist_new() { whiptail_install_type_dist_existing() { [ -n "$TESTING" ] && return + + local node_msg + read -r -d '' node_msg <<- EOM + Choose a distributed node type to join to an existing grid. - install_type=$(whiptail --title "$whiptail_title" --radiolist \ - "Choose distributed node type to join to an existing grid. See https://docs.securityonion.net/architecture for details." 14 57 4 \ + See https://docs.securityonion.net/architecture for details. + + Note: Heavy nodes (HEAVYNODE) are NOT recommended for most users. + EOM + + install_type=$(whiptail --title "$whiptail_title" --radiolist "$node_msg" 17 57 4 \ "SENSOR" "Create a forward only sensor " ON \ "SEARCHNODE" "Add a search node with parsing " OFF \ "FLEET" "Dedicated Fleet Osquery Node " OFF \ From 6c16d6d2223578a171f4309a809c91d810db91c1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Nov 2021 11:15:28 -0500 Subject: [PATCH 105/110] Update invalid hostname message --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 3f8628a30..13bfa82b4 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -908,7 +908,7 @@ whiptail_invalid_hostname() { local error_message read -r -d '' error_message <<- EOM - Please choose a valid hostname. It cannot be localhost; and must contain only the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', and hyphen ('-') + Please choose a valid hostname. It cannot be localhost. It must contain only the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', and hyphen ('-'). EOM if [[ $is_manager_hostname = 0 ]]; then From 46d3eb452d8026866a97155c7efdd3f1a9f82e9b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 8 Nov 2021 20:08:56 +0000 Subject: [PATCH 106/110] Add ECS testing pipeline --- salt/elasticsearch/files/ingest/ecs | 155 ++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/ecs diff --git a/salt/elasticsearch/files/ingest/ecs b/salt/elasticsearch/files/ingest/ecs new file mode 100644 index 000000000..e52ab6e71 --- /dev/null +++ b/salt/elasticsearch/files/ingest/ecs @@ -0,0 +1,155 @@ +{ + "description" : "ECS Testing Pipeline", + "processors": [ + { + "append": { + "field": "event.category", + "value": [ + "process" + ], + "if": "ctx?.wazuh?.data?.type == 'process'", + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "event.type", + "value": [ + "start" + ], + "if": "ctx?.wazuh?.data?.type == 'process'", + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "event.type", + "value": "end", + "if": "ctx?.wazuh?.data?.type == 'process_end'", + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "user.name", + "copy_from": "process.user", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "host.os.type", + "copy_from": "wazuh.data.os.sysname", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "host.os.platform", + "copy_from": "wazuh.data.os.platform", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "host.os.name", + "copy_from": "wazuh.data.os.name", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "host.os.version", + "copy_from": "wazuh.data.os.version", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "signal.rule.name", + "copy_from": "rule.name", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "signal.rule.type", + "copy_from": "rule.category", + "ignore_empty_value": true, + "ignore_failure": true + } + }, + { + "set": { + "field": "signal.rule.threat.tactic.name", + "copy_from": "rule.mitre.tactic", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true + } + }, + { + "append": { + "field": "event.category", + "value": [ + "authentication" + ], + "if": "if(ctx?.rule?.groups != null) {\n if(ctx?.rule?.groups?.contains('authentication_success')) {\n return true\n }\n if(ctx?.rule?.groups?.contains('authentication_failed')) {\n return true\n }\n return false\n}", + "ignore_failure": true + } + }, + { + "set": { + "field": "event.outcome", + "value": "success", + "ignore_empty_value": true, + "if": "ctx?.rule?.groups != null && ctx?.rule?.groups.contains('authentication_success')", + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "event.outcome", + "value": "failure", + "ignore_empty_value": true, + "if": "ctx?.rule?.groups != null && ctx?.rule?.groups.contains('authentication_failed')", + "tag": "test", + "ignore_failure": true + } + }, + { + "set": { + "field": "url.path", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true, + "copy_from": "url.original" + } + }, + { + "set": { + "field": "url.domain", + "ignore_empty_value": true, + "tag": "test", + "ignore_failure": true, + "copy_from": "kibana.log.meta.req.headers.origin" + } + } + ] +} From a8c02252dce77203458d6d882e1c20b04e6126e7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 8 Nov 2021 15:16:05 -0500 Subject: [PATCH 107/110] Update acng.conf --- salt/manager/files/acng/acng.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf index 993452b57..1b7f05e04 100644 --- a/salt/manager/files/acng/acng.conf +++ b/salt/manager/files/acng/acng.conf @@ -77,7 +77,7 @@ FreshIndexMaxAge: 300 # AllowUserPorts: 80 RedirMax: 6 # VfileUseRangeOps is set for fedora volatile files on mirrors that dont to range -VfileUseRangeOps: 0 +VfileUseRangeOps: -1 # PassThroughPattern: private-ppa\.launchpad\.net:443$ # PassThroughPattern: .* # this would allow CONNECT to everything PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ From eefc9cfcb6c2898911d5f01c79398c24237771fe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 8 Nov 2021 16:50:43 -0500 Subject: [PATCH 108/110] Enable Subject Alt Name for registry --- salt/ssl/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index a9aa66703..da71cc708 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -269,6 +269,7 @@ registry_crt: x509.certificate_managed: - name: /etc/pki/registry.crt - ca_server: {{ ca_server }} + - subjectAltName: DNS:{{ manager }}, IP:{{ managerip }} - signing_policy: registry - public_key: /etc/pki/registry.key - CN: {{ manager }} From ea7289d92e752c0f607718b7b1cae34fe8b3124e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 9 Nov 2021 08:20:19 -0500 Subject: [PATCH 109/110] Fix preflight script on centos --- setup/so-preflight | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup/so-preflight b/setup/so-preflight index fd6b5f736..c0e2db135 100755 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -35,14 +35,21 @@ check_default_repos() { echo "" >> "$preflight_log" yum -y check-update >> $preflight_log 2>&1 ret_code=$? + if [[ $ret_code == 0 || $ret_code == 100 ]]; then + printf '%s\n' ' SUCCESS' + ret_code=0 + else + printf '%s\n' ' FAILURE' + fi else printf '%s' 'apt update.' | tee -a "$preflight_log" echo "" >> "$preflight_log" retry 50 10 "apt-get -y update" >> $preflight_log 2>&1 ret_code=$? + [[ $ret_code == 0 ]] && printf '%s\n' ' SUCCESS' || printf '%s\n' ' FAILURE' + fi - [[ $ret_code == 0 ]] && printf '%s\n' ' SUCCESS' || printf '%s\n' ' FAILURE' return $ret_code } From 4bae57d994ac229591bb3036baae01000dafb693 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 9 Nov 2021 08:34:02 -0500 Subject: [PATCH 110/110] Fix preflight printing to log --- setup/so-preflight | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/setup/so-preflight b/setup/so-preflight index c0e2db135..756c721dc 100755 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -28,10 +28,19 @@ fi check_default_repos() { local ret_code=0 + local repo_str=' Checking OS default repos with ' + if [[ $script_run == true ]]; then + printf '%s' "$repo_str" + else + printf '%s' "$repo_str" | tee -a "$preflight_log" + fi - printf ' Checking OS default repos with ' | tee -a "$preflight_log" if [[ $OS == 'centos' ]]; then - printf '%s' 'yum update.' | tee -a "$preflight_log" + if [[ $script_run == true ]]; then + printf '%s' 'yum update.' + else + printf '%s' 'yum update.' | tee -a "$preflight_log" + fi echo "" >> "$preflight_log" yum -y check-update >> $preflight_log 2>&1 ret_code=$? @@ -42,7 +51,11 @@ check_default_repos() { printf '%s\n' ' FAILURE' fi else - printf '%s' 'apt update.' | tee -a "$preflight_log" + if [[ $script_run == true ]]; then + printf '%s' 'apt update.' + else + printf '%s' 'apt update.' | tee -a "$preflight_log" + fi echo "" >> "$preflight_log" retry 50 10 "apt-get -y update" >> $preflight_log 2>&1 ret_code=$? @@ -54,7 +67,12 @@ check_default_repos() { } check_new_repos() { - printf ' Checking repo URLs added by setup.' | tee -a "$preflight_log" + local repo_url_str=' Checking repo URLs added by setup.' + if [[ $script_run == true ]]; then + printf '%s' "$repo_url_str" + else + printf '%s' "$repo_url_str" | tee -a "$preflight_log" + fi if [[ $OS == 'centos' ]]; then local repo_arr=( @@ -83,7 +101,12 @@ check_new_repos() { } check_misc_urls() { - printf ' Checking various other URLs used by setup.' | tee -a "$preflight_log" + local misc_url_str=' Checking various other URLs used by setup.' + if [[ $script_run == true ]]; then + printf '%s' "$misc_url_str" + else + printf '%s' "$misc_url_str" | tee -a "$preflight_log" + fi local so_version so_version=$(cat ../VERSION)