From 11c641fd1bc9b49b4a5e13b219ee2172e6e6d479 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 27 May 2020 15:34:14 -0400 Subject: [PATCH 01/30] Initial support - integrated Fleet setup --- .../tools/sbin}/so-fleet-setup | 26 +++++++-------- salt/fleet/event_gen-packages.sls | 2 ++ salt/fleet/files/scripts/so-fleet-packages | 33 ------------------- salt/fleet/init.sls | 17 ---------- salt/reactor/fleet.sls | 5 +-- setup/so-setup | 8 ++++- 6 files changed, 25 insertions(+), 66 deletions(-) rename salt/{fleet/files/scripts => common/tools/sbin}/so-fleet-setup (72%) delete mode 100644 salt/fleet/files/scripts/so-fleet-packages diff --git a/salt/fleet/files/scripts/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup similarity index 72% rename from salt/fleet/files/scripts/so-fleet-setup rename to salt/common/tools/sbin/so-fleet-setup index 96ddd5156..93c1c9d74 100644 --- a/salt/fleet/files/scripts/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -2,24 +2,24 @@ {% set MAIN_HOSTNAME = salt['grains.get']('host') %} {% set MAIN_IP = salt['pillar.get']('node:mainip') %} -#so-fleet-setup.sh $FleetEmail +#so-fleet-setup.sh $FleetEmail $FleetPassword -# Enable Fleet -echo "Starting Docker Containers..." -salt-call state.apply mysql queue=True >> /root/fleet-setup.log -salt-call state.apply fleet queue=True >> /root/fleet-setup.log -salt-call state.apply redis queue=True >> /root/fleet-setup.log - -if [ ! "$(docker ps -q -f name=so-fleet)" ]; then - echo "so-fleet container not running... Exiting..." - exit 1 +if [[ $# -ne 2 ]] ; then + echo "Username or Password was not set - exiting now." + exit 1 fi -initpw=$(date +%s | sha256sum | base64 | head -c 16 ; echo) +# Checking to see if required containers are started... +if [ ! "$(docker ps -q -f name=so-fleet)" ]; then + echo "Starting Docker Containers..." + salt-call state.apply mysql queue=True >> /root/fleet-setup.log + salt-call state.apply fleet queue=True >> /root/fleet-setup.log + salt-call state.apply redis queue=True >> /root/fleet-setup.log +fi docker exec so-fleet /bin/ash -c "echo {{ MAIN_IP }} {{ MAIN_HOSTNAME }} >> /etc/hosts" docker exec so-fleet fleetctl config set --address https://{{ MAIN_HOSTNAME }}:443 --tls-skip-verify --url-prefix /fleet -docker exec so-fleet fleetctl setup --email $1 --password $initpw +docker exec so-fleet fleetctl setup --email $1 --password $2 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/Windows/osquery.yaml @@ -45,4 +45,4 @@ docker stop so-nginx salt-call state.apply nginx queue=True >> /root/fleet-setup.log echo "Fleet Setup Complete - Login here: https://{{ MAIN_HOSTNAME }}" -echo "Your username is $2 and your password is $initpw" +echo "Your username is $1 and your password is $2" diff --git a/salt/fleet/event_gen-packages.sls b/salt/fleet/event_gen-packages.sls index 3119ada51..11a3d9a0a 100644 --- a/salt/fleet/event_gen-packages.sls +++ b/salt/fleet/event_gen-packages.sls @@ -1,6 +1,7 @@ {% set MASTER = salt['grains.get']('master') %} {% set ENROLLSECRET = salt['pillar.get']('secrets:fleet_enroll-secret') %} {% set CURRENTPACKAGEVERSION = salt['pillar.get']('static:fleet_packages-version') %} +{% set VERSION = salt['pillar.get']('static:soversion') %} so/fleet: event.send: @@ -12,4 +13,5 @@ so/fleet: enroll-secret: {{ ENROLLSECRET }} current-package-version: {{ CURRENTPACKAGEVERSION }} master: {{ MASTER }} + version: {{ VERSION }} \ No newline at end of file diff --git a/salt/fleet/files/scripts/so-fleet-packages b/salt/fleet/files/scripts/so-fleet-packages deleted file mode 100644 index 49f3eebcd..000000000 --- a/salt/fleet/files/scripts/so-fleet-packages +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -{% set MAIN_HOSTNAME = salt['grains.get']('host') %} -{% set MAIN_IP = salt['pillar.get']('node:mainip') %} - - -#so-fleet-packages $FleetHostname/IP - -#if [ ! "$(docker ps -q -f name=so-fleet)" ]; then -# echo "so-fleet container not running... Exiting..." -# exit 1 -#fi - -#docker exec so-fleet /bin/ash -c "echo {{ MAIN_IP }} {{ MAIN_HOSTNAME }} >> /etc/hosts" -#esecret=$(docker exec so-fleet fleetctl get enroll-secret) - -#Concat fleet.crt & ca.crt - this is required for launcher connectivity -#cat /etc/pki/fleet.crt /etc/pki/ca.crt > /etc/pki/launcher.crt -#Actually only need to use /etc/ssl/certs/intca.crt - -#Create the output directory -#mkdir /opt/so/conf/fleet/packages - -docker run \ - --rm \ - --mount type=bind,source=/opt/so/conf/fleet/packages,target=/output \ - --mount type=bind,source=/etc/ssl/certs/intca.crt,target=/var/launcher/launcher.crt \ - docker.io/soshybridhunter/so-fleet-launcher:HH1.1.0 "$esecret" "$1":8090 - -cp /opt/so/conf/fleet/packages/launcher.* /opt/so/saltstack/salt/launcher/packages/ - -#Update timestamp on packages webpage -sed -i "s@.*Generated.*@Generated: $(date '+%m%d%Y')@g" /opt/so/conf/fleet/packages/index.html -sed -i "s@.*Generated.*@Generated: $(date '+%m%d%Y')@g" /opt/so/saltstack/salt/fleet/files/dedicated-index.html \ No newline at end of file diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 7785a3c20..6a6eb9a30 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -16,14 +16,6 @@ include: - mysql -#{% if grains.id.split('_')|last in ['master', 'eval', 'fleet'] %} -#so/fleet: -# event.send: -# - data: -# action: 'enablefleet' -# hostname: {{ grains.host }} -#{% endif %} - # Fleet Setup fleetcdir: file.directory: @@ -67,15 +59,6 @@ fleetlogdir: - group: 939 - makedirs: True -fleetsetupscripts: - file.recurse: - - name: /usr/sbin - - user: 0 - - group: 0 - - file_mode: 755 - - template: jinja - - source: salt://fleet/files/scripts - osquerypackageswebpage: file.managed: - name: /opt/so/conf/fleet/packages/index.html diff --git a/salt/reactor/fleet.sls b/salt/reactor/fleet.sls index d841d54d6..ac4bd422f 100644 --- a/salt/reactor/fleet.sls +++ b/salt/reactor/fleet.sls @@ -13,11 +13,12 @@ def run(): ROLE = data['data']['role'] ESECRET = data['data']['enroll-secret'] MAINIP = data['data']['mainip'] + VERSION = data['data']['version'] STATICFILE = '/opt/so/saltstack/pillar/static.sls' SECRETSFILE = '/opt/so/saltstack/pillar/secrets.sls' - if MINIONID.split('_')[-1] in ['master','eval','fleet','mastersearch']: + if MINIONID.split('_')[-1] in ['master','eval','fleet','mastersearch','standalone']: if ACTION == 'enablefleet': logging.info('so/fleet enablefleet reactor') @@ -55,7 +56,7 @@ def run(): # Run Docker container that will build the packages gen_packages = subprocess.run(["docker", "run","--rm", "--mount", "type=bind,source=/opt/so/saltstack/salt/fleet/packages,target=/output", \ - "--mount", "type=bind,source=/etc/ssl/certs/intca.crt,target=/var/launcher/launcher.crt", f"{ MASTER }:5000/soshybridhunter/so-fleet-launcher:HH1.3.0", \ + "--mount", "type=bind,source=/etc/ssl/certs/intca.crt,target=/var/launcher/launcher.crt", f"{ MASTER }:5000/soshybridhunter/so-fleet-launcher:{ VERSION }", \ f"{ESECRET}", f"{HOSTNAME}:8090", f"{PACKAGEVERSION}.1.1"], stdout=subprocess.PIPE, encoding='ascii') # Update the 'packages-built' timestamp on the webpage (stored in the static pillar) diff --git a/setup/so-setup b/setup/so-setup index 4e004b425..fc934e8a3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -473,11 +473,17 @@ fi fi if [[ "$OSQUERY" = 1 ]]; then + set_progress_str 73 "$(print_salt_state_apply 'mysql')" + salt-call state.apply -l info mysql >> $setup_log 2>&1 + set_progress_str 73 "$(print_salt_state_apply 'fleet')" salt-call state.apply -l info fleet >> $setup_log 2>&1 - set_progress_str 74 "$(print_salt_state_apply 'redis')" + set_progress_str 73 "$(print_salt_state_apply 'redis')" salt-call state.apply -l info redis >> $setup_log 2>&1 + + set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" + so-fleet-setup $WEBUSER $WEBPASSWD1 >> $setup_log 2>&1 fi if [[ "$WAZUH" = 1 ]]; then From e53e891bd64cb88606cdaced256554ad557fecfb Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 28 May 2020 11:36:38 -0400 Subject: [PATCH 02/30] Fleet reactor - Typo fix --- salt/common/tools/sbin/so-fleet-setup | 3 +-- salt/reactor/fleet.sls | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup index 93c1c9d74..810b613c3 100644 --- a/salt/common/tools/sbin/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -44,5 +44,4 @@ salt-call state.apply filebeat queue=True >> /root/fleet-setup.log docker stop so-nginx salt-call state.apply nginx queue=True >> /root/fleet-setup.log -echo "Fleet Setup Complete - Login here: https://{{ MAIN_HOSTNAME }}" -echo "Your username is $1 and your password is $2" +echo "Fleet Setup Complete - Login here with the username and password you ran the script with: https://{{ MAIN_HOSTNAME }}" diff --git a/salt/reactor/fleet.sls b/salt/reactor/fleet.sls index ac4bd422f..2e929c59e 100644 --- a/salt/reactor/fleet.sls +++ b/salt/reactor/fleet.sls @@ -13,7 +13,6 @@ def run(): ROLE = data['data']['role'] ESECRET = data['data']['enroll-secret'] MAINIP = data['data']['mainip'] - VERSION = data['data']['version'] STATICFILE = '/opt/so/saltstack/pillar/static.sls' SECRETSFILE = '/opt/so/saltstack/pillar/secrets.sls' @@ -50,6 +49,7 @@ def run(): PACKAGEVERSION = data['data']['current-package-version'] MASTER = data['data']['master'] + VERSION = data['data']['version'] # Increment the package version by 1 PACKAGEVERSION += 1 From aeb71bb8f0597bdf0f59b9ea1cb02726cde7f95c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 28 May 2020 13:21:25 -0400 Subject: [PATCH 03/30] Simplified setup script --- salt/common/tools/sbin/so-fleet-setup | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup index 810b613c3..956a8b829 100644 --- a/salt/common/tools/sbin/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -1,8 +1,6 @@ #!/bin/bash -{% set MAIN_HOSTNAME = salt['grains.get']('host') %} -{% set MAIN_IP = salt['pillar.get']('node:mainip') %} -#so-fleet-setup.sh $FleetEmail $FleetPassword +#so-fleet-setup $FleetEmail $FleetPassword if [[ $# -ne 2 ]] ; then echo "Username or Password was not set - exiting now." @@ -17,8 +15,7 @@ if [ ! "$(docker ps -q -f name=so-fleet)" ]; then salt-call state.apply redis queue=True >> /root/fleet-setup.log fi -docker exec so-fleet /bin/ash -c "echo {{ MAIN_IP }} {{ MAIN_HOSTNAME }} >> /etc/hosts" -docker exec so-fleet fleetctl config set --address https://{{ MAIN_HOSTNAME }}:443 --tls-skip-verify --url-prefix /fleet +docker exec so-fleet fleetctl config set --address https://localhost:8080 --tls-skip-verify --url-prefix /fleet docker exec so-fleet fleetctl setup --email $1 --password $2 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml @@ -44,4 +41,4 @@ salt-call state.apply filebeat queue=True >> /root/fleet-setup.log docker stop so-nginx salt-call state.apply nginx queue=True >> /root/fleet-setup.log -echo "Fleet Setup Complete - Login here with the username and password you ran the script with: https://{{ MAIN_HOSTNAME }}" +echo "Fleet Setup Complete - Login with the username and password you ran the script with." From 0d8c0e1fa669d74b9c14c15a6ab225dc47bc7786 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 28 May 2020 16:27:18 -0400 Subject: [PATCH 04/30] Fleet standalone fixes --- setup/so-functions | 28 ++++++++++++++++++++++++++++ setup/so-setup | 11 +++++++++-- setup/so-whiptail | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 612b0147b..462d3b64c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -246,6 +246,10 @@ check_soremote_pass() { check_pass_match "$SOREMOTEPASS1" "$SOREMOTEPASS2" "SCMATCH" } +check_fleet_node_pass() { + check_pass_match "$FLEETNODEPASSWD1" "$FLEETNODEPASSWD2" "fpMATCH" +} + check_web_pass() { check_pass_match "$WEBPASSWD1" "$WEBPASSWD2" "WPMATCH" } @@ -283,6 +287,30 @@ collect_adminuser_inputs() { done } +collect_fleetuser_inputs() { + # Get a username & password for the Fleet admin user + local valid_user=no + while [[ $valid_user != yes ]]; do + whiptail_create_fleet_node_user + if so-user valemail "$FLEETNODEUSER" >> "$setup_log" 2>&1; then + valid_user=yes + else + whiptail_invalid_user_warning + fi + done + + FPMATCH=no + while [[ $FPMATCH != yes ]]; do + whiptail_create_fleet_node_user_password1 + if echo "$FLEETNODEPASSWD1" | so-user valpass >> "$setup_log" 2>&1; then + whiptail_create_fleet_node_user_password2 + check_fleet_node_pass + else + whiptail_invalid_pass_warning + fi + done +} + collect_webuser_inputs() { # Get a password for the web admin user diff --git a/setup/so-setup b/setup/so-setup index fc934e8a3..69f46184d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -481,9 +481,16 @@ fi set_progress_str 73 "$(print_salt_state_apply 'redis')" salt-call state.apply -l info redis >> $setup_log 2>&1 + + if [ "$install_type" = 'FLEET' ]; then + collect_fleetuser_inputs + set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" + so-fleet-setup $FLEETNODEUSER $FLEETNODEPASSWD1 >> $setup_log 2>&1 + else + set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" + so-fleet-setup $WEBUSER $WEBPASSWD1 >> $setup_log 2>&1 + fi - set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" - so-fleet-setup $WEBUSER $WEBPASSWD1 >> $setup_log 2>&1 fi if [[ "$WAZUH" = 1 ]]; then diff --git a/setup/so-whiptail b/setup/so-whiptail index 31a0c05ec..e9f691b55 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -165,6 +165,38 @@ whiptail_create_admin_user_password2() { } +whiptail_create_fleet_node_user() { + + [ -n "$TESTING" ] && return + + FLEETNODEUSER=$(whiptail --title "Security Onion Install" --inputbox \ + "Please enter an email as the username for the Fleet admin user." 10 60 3>&1 1>&2 2>&3) + +} + +whiptail_create_fleet_node_user_password1() { + + [ -n "$TESTING" ] && return + + FLEETNODEPASS1=$(whiptail --title "Security Onion Install" --passwordbox \ + "Enter a password for $FLEETNODEUSER" 10 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + +whiptail_create_fleet_node_user_password2() { + + [ -n "$TESTING" ] && return + + FLEETNODEPASS2=$(whiptail --title "Security Onion Install" --passwordbox \ + "Re-enter a password for $FLEETNODEUSER" 10 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + whiptail_create_soremote_user() { [ -n "$TESTING" ] && return From e8e48a4bebae5bd8c6edb70baf834d920019e4cf Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 28 May 2020 16:40:12 -0400 Subject: [PATCH 05/30] Fleet standalone bugfixes --- setup/so-setup | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 69f46184d..5906eefec 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -283,6 +283,13 @@ if [[ $is_node && ! $is_eval ]]; then fi fi +if [ "$install_type" = 'FLEET' ]; then + collect_fleetuser_inputs +else + FLEETNODEUSER=$WEBUSER + FLEETNODEPASSWD1=$WEBPASSWD1 +fi + whiptail_make_changes if [[ -n "$TURBO" ]]; then @@ -482,14 +489,8 @@ fi set_progress_str 73 "$(print_salt_state_apply 'redis')" salt-call state.apply -l info redis >> $setup_log 2>&1 - if [ "$install_type" = 'FLEET' ]; then - collect_fleetuser_inputs - set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" - so-fleet-setup $FLEETNODEUSER $FLEETNODEPASSWD1 >> $setup_log 2>&1 - else - set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" - so-fleet-setup $WEBUSER $WEBPASSWD1 >> $setup_log 2>&1 - fi + set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" + so-fleet-setup $FLEETNODEUSER $FLEETNODEPASSWD1 >> $setup_log 2>&1 fi From 046bca626e760a33e50dc2c1c2a5fba1d1835912 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 28 May 2020 16:51:37 -0400 Subject: [PATCH 06/30] Fleet pw check fix --- setup/so-functions | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 462d3b64c..e487e5542 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -247,7 +247,7 @@ check_soremote_pass() { } check_fleet_node_pass() { - check_pass_match "$FLEETNODEPASSWD1" "$FLEETNODEPASSWD2" "fpMATCH" + check_pass_match "$FLEETNODEPASSWD1" "$FLEETNODEPASSWD2" "FPMATCH" } check_web_pass() { @@ -302,12 +302,8 @@ collect_fleetuser_inputs() { FPMATCH=no while [[ $FPMATCH != yes ]]; do whiptail_create_fleet_node_user_password1 - if echo "$FLEETNODEPASSWD1" | so-user valpass >> "$setup_log" 2>&1; then - whiptail_create_fleet_node_user_password2 - check_fleet_node_pass - else - whiptail_invalid_pass_warning - fi + whiptail_create_fleet_node_user_password2 + check_fleet_node_pass done } From 4b14ecf1d92cfe9aece77bef989be5b974932c03 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 1 Jun 2020 16:36:32 -0400 Subject: [PATCH 07/30] Fleet standalone fixes --- salt/fleet/event_enable-fleet.sls | 3 ++- salt/mysql/init.sls | 3 ++- salt/nginx/etc/nginx.conf.so-fleet | 4 +++- salt/ssl/init.sls | 3 ++- setup/so-functions | 11 +++++++++++ setup/so-setup | 7 ++++++- setup/so-whiptail | 2 +- 7 files changed, 27 insertions(+), 6 deletions(-) diff --git a/salt/fleet/event_enable-fleet.sls b/salt/fleet/event_enable-fleet.sls index 007f3690c..90bfec2d4 100644 --- a/salt/fleet/event_enable-fleet.sls +++ b/salt/fleet/event_enable-fleet.sls @@ -1,5 +1,6 @@ {% set ENROLLSECRET = salt['cmd.run']('docker exec so-fleet fleetctl get enroll-secret') %} -{%- set MAINIP = salt['pillar.get']('node:mainip') -%} +{% set MAININT = salt['pillar.get']('host:mainint') %} +{% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} so/fleet: event.send: diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index f3ce61784..c96be214c 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -6,7 +6,8 @@ {% set FLEETARCH = salt['grains.get']('role') %} {% if FLEETARCH == "so-fleet" %} - {% set MAINIP = salt['pillar.get']('node:mainip') %} + {% set MAININT = salt['pillar.get']('host:mainint') %} + {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% else %} {% set MAINIP = salt['pillar.get']('static:masterip') %} {% endif %} diff --git a/salt/nginx/etc/nginx.conf.so-fleet b/salt/nginx/etc/nginx.conf.so-fleet index 28372f448..7d6974532 100644 --- a/salt/nginx/etc/nginx.conf.so-fleet +++ b/salt/nginx/etc/nginx.conf.so-fleet @@ -1,4 +1,6 @@ -{%- set MAINIP = salt['pillar.get']('node:mainip', '') %} +{% set MAININT = salt['pillar.get']('host:mainint') %} +{% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} + # For more information on configuration, see: # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 384c5bd5f..ca6417dec 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -1,9 +1,10 @@ {% set master = salt['grains.get']('master') %} {% set masterip = salt['pillar.get']('static:masterip', '') %} {% set HOSTNAME = salt['grains.get']('host') %} -{% set MAINIP = salt['pillar.get']('node:mainip') %} {% set global_ca_text = [] %} {% set global_ca_server = [] %} +{% set MAININT = salt['pillar.get']('host:mainint') %} +{% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% if grains.id.split('_')|last in ['master', 'eval', 'standalone'] %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} diff --git a/setup/so-functions b/setup/so-functions index e487e5542..3aea34bac 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -810,6 +810,17 @@ get_minion_type() { echo "$minion_type" } +host_pillar() { + + local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls + + # Create the host pillar + printf '%s\n'\ + "host:"\ + " mainint: $MNIC"\ + "" > "$pillar_file" +} + install_cleanup() { echo "Installer removing the following files:" ls -lR "$temp_install_dir" diff --git a/setup/so-setup b/setup/so-setup index 5906eefec..313586800 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -128,6 +128,7 @@ elif [ "$install_type" = 'HEAVYNODE' ]; then is_sensor=true elif [ "$install_type" = 'FLEET' ]; then is_minion=true + is_fleet_standalone=true OSQUERY=1 elif [ "$install_type" = 'HELIXSENSOR' ]; then is_helix=true @@ -234,7 +235,7 @@ if [[ $is_master ]]; then get_redirect fi -if [[ $is_distmaster || ( $is_sensor || $is_node ) && ! $is_eval ]]; then +if [[ $is_distmaster || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ]]; then whiptail_master_updates if [[ $setup_type == 'network' && $MASTERUPDATES == 1 ]]; then whiptail_master_updates_warning @@ -326,6 +327,10 @@ if [[ $is_minion ]]; then copy_ssh_key >> $setup_log 2>&1 fi +if [[ $is_fleet_standalone ]]; then + host_pillar >> $setup_log 2>&1 +fi + # Begin install { # Set initial percentage to 0 diff --git a/setup/so-whiptail b/setup/so-whiptail index e9f691b55..06712ace3 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -170,7 +170,7 @@ whiptail_create_fleet_node_user() { [ -n "$TESTING" ] && return FLEETNODEUSER=$(whiptail --title "Security Onion Install" --inputbox \ - "Please enter an email as the username for the Fleet admin user." 10 60 3>&1 1>&2 2>&3) + "Please enter an email for use as the username for the Fleet admin user." 10 60 3>&1 1>&2 2>&3) } From b695b7f24515c0b5ef8edf9e941dc327c9e1be50 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Jun 2020 08:05:48 -0400 Subject: [PATCH 08/30] Fleet standalone fixes - firewall --- salt/firewall/init.sls | 3 ++- setup/so-functions | 3 +++ setup/so-setup | 4 +++- setup/so-whiptail | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index b6c928eba..c27188f12 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -6,7 +6,8 @@ {% elif grains['role'] == 'so-sensor' %} {% set ip = salt['pillar.get']('sensor:mainip', '') %} {% elif grains['role'] == 'so-fleet' %} - {% set ip = salt['pillar.get']('node:mainip', '') %} + {% set MAININT = salt['pillar.get']('host:mainint') %} + {% set ip = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% endif %} {% set FLEET_NODE = salt['pillar.get']('static:fleet_node') %} diff --git a/setup/so-functions b/setup/so-functions index 3aea34bac..55cb71327 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -398,6 +398,9 @@ check_requirements() { req_mem=12 req_cores=4 req_nics=2 + elif [[ "$eval_or_dist" == 'fleet' ]]; then + req_mem=4 + req_cores=4 elif [[ "$eval_or_dist" == 'dist' ]]; then req_mem=8 req_cores=4 diff --git a/setup/so-setup b/setup/so-setup index 313586800..f5d43eb38 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -138,6 +138,8 @@ if [[ $is_eval ]]; then check_requirements "eval" elif [[ $is_distmaster || $is_minion ]]; then check_requirements "dist" +elif [[ $is_fleet_standalone ]]; then + check_requirements "fleet" elif [[ $is_sensor && ! $is_eval ]]; then check_requirements "dist" "sensor" fi @@ -284,7 +286,7 @@ if [[ $is_node && ! $is_eval ]]; then fi fi -if [ "$install_type" = 'FLEET' ]; then +if [ "$install_type" == 'FLEET' ]; then collect_fleetuser_inputs else FLEETNODEUSER=$WEBUSER diff --git a/setup/so-whiptail b/setup/so-whiptail index 06712ace3..fc2429ad2 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -178,7 +178,7 @@ whiptail_create_fleet_node_user_password1() { [ -n "$TESTING" ] && return - FLEETNODEPASS1=$(whiptail --title "Security Onion Install" --passwordbox \ + FLEETNODEPASSWD1=$(whiptail --title "Security Onion Install" --passwordbox \ "Enter a password for $FLEETNODEUSER" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -189,7 +189,7 @@ whiptail_create_fleet_node_user_password2() { [ -n "$TESTING" ] && return - FLEETNODEPASS2=$(whiptail --title "Security Onion Install" --passwordbox \ + FLEETNODEPASSWD2=$(whiptail --title "Security Onion Install" --passwordbox \ "Re-enter a password for $FLEETNODEUSER" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? From b5cc653179e45cf363a909a94fcdc3cf6a396ca2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Jun 2020 09:39:42 -0400 Subject: [PATCH 09/30] Fleet standalone fixes - mainip --- salt/fleet/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 6a6eb9a30..a4998047e 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -3,12 +3,12 @@ {%- set FLEETJWT = salt['pillar.get']('secrets:fleet_jwt', None) -%} {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} {% set MASTER = salt['grains.get']('master') %} -{% set MAINIP = salt['pillar.get']('node:mainip') %} {% set FLEETARCH = salt['grains.get']('role') %} {% if FLEETARCH == "so-fleet" %} - {% set MAINIP = salt['pillar.get']('node:mainip') %} + {% set MAININT = salt['pillar.get']('host:mainint') %} + {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% else %} {% set MAINIP = salt['pillar.get']('static:masterip') %} {% endif %} From 77df87880ceab8fd06d12e8462438281202b1aee Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Jun 2020 10:20:29 -0400 Subject: [PATCH 10/30] Fleet standalone fixes - fleet sa req --- setup/so-functions | 4 +--- setup/so-setup | 9 ++++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 55cb71327..f3e738678 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -398,13 +398,11 @@ check_requirements() { req_mem=12 req_cores=4 req_nics=2 - elif [[ "$eval_or_dist" == 'fleet' ]]; then - req_mem=4 - req_cores=4 elif [[ "$eval_or_dist" == 'dist' ]]; then req_mem=8 req_cores=4 if [[ "$node_type" == 'sensor' ]]; then req_nics=2; else req_nics=1; fi + if [[ "$node_type" == 'fleet' ]]; then req_mem=4; fi fi if [[ $num_nics -lt $req_nics ]]; then diff --git a/setup/so-setup b/setup/so-setup index f5d43eb38..45452a0a7 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -128,7 +128,6 @@ elif [ "$install_type" = 'HEAVYNODE' ]; then is_sensor=true elif [ "$install_type" = 'FLEET' ]; then is_minion=true - is_fleet_standalone=true OSQUERY=1 elif [ "$install_type" = 'HELIXSENSOR' ]; then is_helix=true @@ -136,10 +135,10 @@ fi if [[ $is_eval ]]; then check_requirements "eval" +elif [[ "$is_minon" && "$OSQUERY" == 1 ]]; then + check_requirements "dist" "fleet" elif [[ $is_distmaster || $is_minion ]]; then check_requirements "dist" -elif [[ $is_fleet_standalone ]]; then - check_requirements "fleet" elif [[ $is_sensor && ! $is_eval ]]; then check_requirements "dist" "sensor" fi @@ -237,7 +236,7 @@ if [[ $is_master ]]; then get_redirect fi -if [[ $is_distmaster || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ]]; then +if [[ $is_distmaster || ( $is_sensor || $is_node || [[ "$is_minon" && "$OSQUERY" == 1 ]] ) && ! $is_eval ]]; then whiptail_master_updates if [[ $setup_type == 'network' && $MASTERUPDATES == 1 ]]; then whiptail_master_updates_warning @@ -329,7 +328,7 @@ if [[ $is_minion ]]; then copy_ssh_key >> $setup_log 2>&1 fi -if [[ $is_fleet_standalone ]]; then +if [[ "$is_minon" && "$OSQUERY" == 1 ]]; then host_pillar >> $setup_log 2>&1 fi From 782c6698358d0cffd406be1128f2972b88666bb1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Jun 2020 12:42:14 -0400 Subject: [PATCH 11/30] Fleet standalone fixes - req --- setup/so-setup | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 45452a0a7..c583c4260 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -128,6 +128,7 @@ elif [ "$install_type" = 'HEAVYNODE' ]; then is_sensor=true elif [ "$install_type" = 'FLEET' ]; then is_minion=true + is_fleet_standalone=true OSQUERY=1 elif [ "$install_type" = 'HELIXSENSOR' ]; then is_helix=true @@ -135,12 +136,12 @@ fi if [[ $is_eval ]]; then check_requirements "eval" -elif [[ "$is_minon" && "$OSQUERY" == 1 ]]; then +elif [[ $is_fleet_standalone ]]; then check_requirements "dist" "fleet" -elif [[ $is_distmaster || $is_minion ]]; then - check_requirements "dist" elif [[ $is_sensor && ! $is_eval ]]; then check_requirements "dist" "sensor" +elif [[ $is_distmaster || $is_minion ]]; then + check_requirements "dist" fi whiptail_patch_schedule @@ -236,7 +237,7 @@ if [[ $is_master ]]; then get_redirect fi -if [[ $is_distmaster || ( $is_sensor || $is_node || [[ "$is_minon" && "$OSQUERY" == 1 ]] ) && ! $is_eval ]]; then +if [[ $is_distmaster || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ]]; then whiptail_master_updates if [[ $setup_type == 'network' && $MASTERUPDATES == 1 ]]; then whiptail_master_updates_warning @@ -328,7 +329,7 @@ if [[ $is_minion ]]; then copy_ssh_key >> $setup_log 2>&1 fi -if [[ "$is_minon" && "$OSQUERY" == 1 ]]; then +if [[ $is_fleet_standalone ]]; then host_pillar >> $setup_log 2>&1 fi From eaacb7b71e6a3ff83063a64dcb9f6c1411345a50 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 3 Jun 2020 05:54:35 -0400 Subject: [PATCH 12/30] Fleet cleanup --- salt/common/tools/sbin/so-fleet-setup | 4 +- salt/firewall/init.sls | 31 +++--- salt/fleet/files/dedicated-index.html | 96 ------------------- salt/fleet/files/packs/PUT.PACKS.IN.HERE | 0 salt/fleet/files/packs/hh/hh-post-login.sh | 13 --- .../{hh/osquery.conf => osquery-config.conf} | 0 .../{hh/hhdefault.yml => so/so-default.yml} | 0 salt/fleet/init.sls | 7 -- 8 files changed, 20 insertions(+), 131 deletions(-) delete mode 100644 salt/fleet/files/dedicated-index.html delete mode 100644 salt/fleet/files/packs/PUT.PACKS.IN.HERE delete mode 100644 salt/fleet/files/packs/hh/hh-post-login.sh rename salt/fleet/files/packs/{hh/osquery.conf => osquery-config.conf} (100%) rename salt/fleet/files/packs/{hh/hhdefault.yml => so/so-default.yml} (100%) diff --git a/salt/common/tools/sbin/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup index 956a8b829..9801c803b 100644 --- a/salt/common/tools/sbin/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -20,9 +20,9 @@ docker exec so-fleet fleetctl setup --email $1 --password $2 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/Windows/osquery.yaml -docker exec so-fleet fleetctl apply -f /packs/hh/hhdefault.yml +docker exec so-fleet fleetctl apply -f /packs/so/so-default.yml docker exec so-fleet /bin/sh -c 'for pack in /packs/palantir/Fleet/Endpoints/packs/*.yaml; do fleetctl apply -f "$pack"; done' -docker exec so-fleet fleetctl apply -f /packs/hh/osquery.conf +docker exec so-fleet fleetctl apply -f /packs/osquery-config.conf # Enable Fleet diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index c27188f12..a91ea20d0 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -355,17 +355,6 @@ enable_minions_influxdb_8086_{{ip}}: - position: 1 - save: True -enable_minion_osquery_8080_{{ip}}: - iptables.insert: - - table: filter - - chain: DOCKER-USER - - jump: ACCEPT - - proto: tcp - - source: {{ ip }} - - dport: 8080 - - position: 1 - - save: True - enable_minion_osquery_8090_{{ip}}: iptables.insert: - table: filter @@ -792,7 +781,7 @@ enable_fleet_osquery_8080_{{ip}}: - save: True -enable_fleetnodetemp_mysql_3306_{{ip}}: +enable_fleetnode_mysql_3306_{{ip}}: iptables.insert: - table: filter - chain: DOCKER-USER @@ -803,7 +792,7 @@ enable_fleetnodetemp_mysql_3306_{{ip}}: - position: 1 - save: True -enable_fleettemp_osquery_8080_{{ip}}: +enable_fleet_osquery_8080_{{ip}}: iptables.insert: - table: filter - chain: DOCKER-USER @@ -847,4 +836,20 @@ enable_fleetnode_8090_{{ip}}: {% endfor %} +# Make it so all the minions can talk to fleet standalone node +{% for ip in pillar.get('minions') %} + +enable_minion_fleet_standalone_8090_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 8090 + - position: 1 + - save: True + +{% endfor %} + {% endif %} diff --git a/salt/fleet/files/dedicated-index.html b/salt/fleet/files/dedicated-index.html deleted file mode 100644 index 4a27b6104..000000000 --- a/salt/fleet/files/dedicated-index.html +++ /dev/null @@ -1,96 +0,0 @@ -{%- set PACKAGESTS = salt['pillar.get']('static:fleet_packages-timestamp:', 'N/A') -%} - - - - -Security Onion - Hybrid Hunter - - - - - - - -
-
- Fleet - Osquery/Fleet Docs - Security Onion Solutions -
- -
-

-

-

Security Onion - Dedicated Fleet Node

-
-
-
-

-
-
- - diff --git a/salt/fleet/files/packs/PUT.PACKS.IN.HERE b/salt/fleet/files/packs/PUT.PACKS.IN.HERE deleted file mode 100644 index e69de29bb..000000000 diff --git a/salt/fleet/files/packs/hh/hh-post-login.sh b/salt/fleet/files/packs/hh/hh-post-login.sh deleted file mode 100644 index cc787decf..000000000 --- a/salt/fleet/files/packs/hh/hh-post-login.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -echo "Applying Post Configuration for Osquery" -#fleetctl apply -f /packs/hh/osquery.conf -fleetctl apply -f /packs/palantir/Fleet/Endpoints/options.yaml -fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml -fleetctl apply -f /packs/palantir/Fleet/Endpoints/Windows/osquery.yaml -fleetctl apply -f /packs/hh/hhdefault.yml - -for pack in /packs/palantir/Fleet/Endpoints/packs/*.yaml; - do fleetctl apply -f "$pack" -done -echo "" -echo "You can now exit the container by typing exit" diff --git a/salt/fleet/files/packs/hh/osquery.conf b/salt/fleet/files/packs/osquery-config.conf similarity index 100% rename from salt/fleet/files/packs/hh/osquery.conf rename to salt/fleet/files/packs/osquery-config.conf diff --git a/salt/fleet/files/packs/hh/hhdefault.yml b/salt/fleet/files/packs/so/so-default.yml similarity index 100% rename from salt/fleet/files/packs/hh/hhdefault.yml rename to salt/fleet/files/packs/so/so-default.yml diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index a4998047e..65f32e213 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -5,7 +5,6 @@ {% set MASTER = salt['grains.get']('master') %} {% set FLEETARCH = salt['grains.get']('role') %} - {% if FLEETARCH == "so-fleet" %} {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} @@ -59,12 +58,6 @@ fleetlogdir: - group: 939 - makedirs: True -osquerypackageswebpage: - file.managed: - - name: /opt/so/conf/fleet/packages/index.html - - source: salt://fleet/files/dedicated-index.html - - template: jinja - fleetdb: mysql_database.present: - name: fleet From 3d92145bb93a2d8d860a7dfc087f848a5343e2ec Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 3 Jun 2020 15:52:18 -0400 Subject: [PATCH 13/30] Fleet standalone - custom hostnames --- salt/fleet/event_gen-packages.sls | 9 +++++++- salt/fleet/event_update-custom-hostname.sls | 9 ++++++++ salt/reactor/fleet.sls | 25 ++++++++++++++++----- salt/ssl/init.sls | 4 +++- setup/so-functions | 4 ++++ setup/so-setup | 7 ++++++ setup/so-whiptail | 13 +++++++++++ 7 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 salt/fleet/event_update-custom-hostname.sls diff --git a/salt/fleet/event_gen-packages.sls b/salt/fleet/event_gen-packages.sls index 11a3d9a0a..72c02be45 100644 --- a/salt/fleet/event_gen-packages.sls +++ b/salt/fleet/event_gen-packages.sls @@ -2,12 +2,19 @@ {% set ENROLLSECRET = salt['pillar.get']('secrets:fleet_enroll-secret') %} {% set CURRENTPACKAGEVERSION = salt['pillar.get']('static:fleet_packages-version') %} {% set VERSION = salt['pillar.get']('static:soversion') %} +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} + +{% if CUSTOM_FLEET_HOSTNAME != None %} + {% set HOSTNAME = {{ CUSTOM_FLEET_HOSTNAME }} %} +{% else %} + {% set HOSTNAME = {{ grains.host }} %} +{% endif %} so/fleet: event.send: - data: action: 'genpackages' - hostname: {{ grains.host }} + package-hostname: {{ HOSTNAME }} role: {{ grains.role }} mainip: {{ grains.host }} enroll-secret: {{ ENROLLSECRET }} diff --git a/salt/fleet/event_update-custom-hostname.sls b/salt/fleet/event_update-custom-hostname.sls new file mode 100644 index 000000000..9278862ed --- /dev/null +++ b/salt/fleet/event_update-custom-hostname.sls @@ -0,0 +1,9 @@ +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} + +so/fleet: + event.send: + - data: + action: 'update_custom_hostname' + custom_hostname: {{ CUSTOM_FLEET_HOSTNAME }} + role: {{ grains.role }} + \ No newline at end of file diff --git a/salt/reactor/fleet.sls b/salt/reactor/fleet.sls index 2e929c59e..c7bade3ab 100644 --- a/salt/reactor/fleet.sls +++ b/salt/reactor/fleet.sls @@ -9,10 +9,6 @@ import subprocess def run(): MINIONID = data['id'] ACTION = data['data']['action'] - HOSTNAME = data['data']['hostname'] - ROLE = data['data']['role'] - ESECRET = data['data']['enroll-secret'] - MAINIP = data['data']['mainip'] STATICFILE = '/opt/so/saltstack/pillar/static.sls' SECRETSFILE = '/opt/so/saltstack/pillar/secrets.sls' @@ -21,6 +17,11 @@ def run(): if ACTION == 'enablefleet': logging.info('so/fleet enablefleet reactor') + ESECRET = data['data']['enroll-secret'] + MAINIP = data['data']['mainip'] + ROLE = data['data']['role'] + HOSTNAME = data['data']['hostname'] + # Enable Fleet for line in fileinput.input(STATICFILE, inplace=True): if ROLE == 'so-fleet': @@ -48,8 +49,10 @@ def run(): logging.info('so/fleet genpackages reactor') PACKAGEVERSION = data['data']['current-package-version'] + PACKAGEHOSTNAME = data['data']['package-hostname'] MASTER = data['data']['master'] VERSION = data['data']['version'] + ESECRET = data['data']['enroll-secret'] # Increment the package version by 1 PACKAGEVERSION += 1 @@ -57,7 +60,7 @@ def run(): # Run Docker container that will build the packages gen_packages = subprocess.run(["docker", "run","--rm", "--mount", "type=bind,source=/opt/so/saltstack/salt/fleet/packages,target=/output", \ "--mount", "type=bind,source=/etc/ssl/certs/intca.crt,target=/var/launcher/launcher.crt", f"{ MASTER }:5000/soshybridhunter/so-fleet-launcher:{ VERSION }", \ - f"{ESECRET}", f"{HOSTNAME}:8090", f"{PACKAGEVERSION}.1.1"], stdout=subprocess.PIPE, encoding='ascii') + f"{ESECRET}", f"{PACKAGEHOSTNAME}:8090", f"{PACKAGEVERSION}.1.1"], stdout=subprocess.PIPE, encoding='ascii') # Update the 'packages-built' timestamp on the webpage (stored in the static pillar) for line in fileinput.input(STATICFILE, inplace=True): @@ -70,6 +73,16 @@ def run(): print(line) # Copy over newly-built packages - copy_packages = subprocess.run(["salt-call", "state.apply","fleet"], stdout=subprocess.PIPE, encoding='ascii') + copy_packages = subprocess.run(["salt-call", "state.apply","fleet"], stdout=subprocess.PIPE, encoding='ascii') + + if ACTION == 'update_custom_hostname': + logging.info('so/fleet update_custom_hostname reactor') + + CUSTOMHOSTNAME = data['data']['custom_hostname'] + + # Update the Fleet host in the static pillar + for line in fileinput.input(STATICFILE, inplace=True): + line = re.sub(r'fleet_custom_hostname: \S*', f"fleet_custom_hostname: {CUSTOMHOSTNAME}", line.rstrip()) + print(line) return {} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index ca6417dec..897ab84d0 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -5,6 +5,7 @@ {% set global_ca_server = [] %} {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} {% if grains.id.split('_')|last in ['master', 'eval', 'standalone'] %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} @@ -201,6 +202,7 @@ chownfilebeatp8: - signing_policy: masterssl - public_key: /etc/pki/masterssl.key - CN: {{ HOSTNAME }} + - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }} {% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }} {% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -223,7 +225,7 @@ chownfilebeatp8: x509.certificate_managed: - signing_private_key: /etc/pki/fleet.key - CN: {{ HOSTNAME }} - - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }} + - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }} {% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }} {% endif %} - days_remaining: 0 - days_valid: 820 - backup: True diff --git a/setup/so-functions b/setup/so-functions index f3e738678..96190b60c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -287,6 +287,10 @@ collect_adminuser_inputs() { done } +collect_fleet_custom_hostname_inputs{ + whiptail_fleet_custom_hostname +} + collect_fleetuser_inputs() { # Get a username & password for the Fleet admin user local valid_user=no diff --git a/setup/so-setup b/setup/so-setup index c583c4260..7f55c1e46 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -288,6 +288,7 @@ fi if [ "$install_type" == 'FLEET' ]; then collect_fleetuser_inputs + collect_fleet_custom_hostname_inputs else FLEETNODEUSER=$WEBUSER FLEETNODEPASSWD1=$WEBPASSWD1 @@ -495,6 +496,12 @@ fi set_progress_str 73 "$(print_salt_state_apply 'redis')" salt-call state.apply -l info redis >> $setup_log 2>&1 + + if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then + set_progress_str 73 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')" + pillar_override="{"static":{"fleet_custom_hostname":"$FLEETCUSTOMHOSTNAME"}}" + salt-call state.apply -l info fleet.event_update-custom-hostname pillar=$pillar_override >> $setup_log 2>&1 + fi set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" so-fleet-setup $FLEETNODEUSER $FLEETNODEPASSWD1 >> $setup_log 2>&1 diff --git a/setup/so-whiptail b/setup/so-whiptail index fc2429ad2..dd6d1edac 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -270,6 +270,19 @@ whiptail_create_web_user_password2() { } +whiptail_fleet_custom_hostname() { + + [ -n "$TESTING" ] && return + + FLEETCUSTOMHOSTNAME=$(whiptail --title "Security Onion Install" --inputbox \ + "What FQDN should osquery clients use for connections to this Fleet node? Leave blank if the local system hostname will be used." 10 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + + + whiptail_requirements_error() { local requirement_needed=$1 From a09991d770504693ddbcbd78d2f9180e43ab2685 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 3 Jun 2020 15:56:59 -0400 Subject: [PATCH 14/30] Fleet standalone - custom hostnames --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index 96190b60c..f300b985d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -918,6 +918,7 @@ master_static() { " cortexorgname: SecurityOnion"\ " cortexorguser: soadmin"\ " cortexorguserkey: $CORTEXORGUSERKEY"\ + " fleet_custom_hostname: "\ " fleet_master: False"\ " fleet_node: False"\ " fleet_packages-timestamp: N/A"\ From 08901715272dd0817500709e3f922f499162438b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 4 Jun 2020 07:51:05 -0400 Subject: [PATCH 15/30] Fixes --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index f300b985d..487b98f0b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -287,7 +287,7 @@ collect_adminuser_inputs() { done } -collect_fleet_custom_hostname_inputs{ +collect_fleet_custom_hostname_inputs() { whiptail_fleet_custom_hostname } From 7edf9a67c3e1f6205a47774c1536df05f6363512 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 4 Jun 2020 10:46:27 -0400 Subject: [PATCH 16/30] Ensure standalone mode supports the settings flow --- salt/nginx/etc/nginx.conf.so-standalone | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/nginx/etc/nginx.conf.so-standalone b/salt/nginx/etc/nginx.conf.so-standalone index 33edb9c3e..2178b6017 100644 --- a/salt/nginx/etc/nginx.conf.so-standalone +++ b/salt/nginx/etc/nginx.conf.so-standalone @@ -134,7 +134,7 @@ http { proxy_set_header Connection "Upgrade"; } - location ~ ^/auth/.*?(whoami|login|logout) { + location ~ ^/auth/.*?(whoami|login|logout|settings) { rewrite /auth/(.*) /$1 break; proxy_pass http://{{ masterip }}:4433; proxy_read_timeout 90; From 4e4eeedb4109a035be794d87ab300c04686d347a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 4 Jun 2020 22:41:35 -0400 Subject: [PATCH 17/30] Initial support - external beats --- pillar/logstash/master.sls | 1 + pillar/logstash/search.sls | 1 - salt/elasticsearch/files/ingest/beats.common | 35 + .../pipelines/config/so/0009_input_beats.conf | 6 + .../config/so/9500_output_beats.conf.jinja | 17 +- .../templates/so/so-beats-template.json | 1286 ----------------- 6 files changed, 47 insertions(+), 1299 deletions(-) create mode 100644 salt/elasticsearch/files/ingest/beats.common create mode 100644 salt/logstash/pipelines/config/so/0009_input_beats.conf delete mode 100644 salt/logstash/pipelines/templates/so/so-beats-template.json diff --git a/pillar/logstash/master.sls b/pillar/logstash/master.sls index 61d1bb6d1..1ff41b43c 100644 --- a/pillar/logstash/master.sls +++ b/pillar/logstash/master.sls @@ -2,5 +2,6 @@ logstash: pipelines: master: config: + - so/0009_input_beats.conf - so/0010_input_hhbeats.conf - so/9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 30bf94cea..6b3d0422e 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -12,6 +12,5 @@ logstash: - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja templates: - - so/so-beats-template.json - so/so-common-template.json - so/so-zeek-template.json diff --git a/salt/elasticsearch/files/ingest/beats.common b/salt/elasticsearch/files/ingest/beats.common new file mode 100644 index 000000000..0e93abb03 --- /dev/null +++ b/salt/elasticsearch/files/ingest/beats.common @@ -0,0 +1,35 @@ +{ + "description" : "beats.common", + "processors" : [ + {"community_id": {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}}, + { "set": { "if": "ctx.winlog?.channel != null", "field": "dataset", "value": "wel-{{winlog.channel}}", "override": true } }, + { "set": { "if": "ctx.agent?.type != null", "field": "module", "value": "{{agent.type}}", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3", "field": "event.category", "value": "host,process,network", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1", "field": "event.category", "value": "host,process", "override": true } }, + { "rename": { "field": "agent.hostname", "target_field": "agent.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationIp", "target_field": "destination.ip", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationPort", "target_field": "destination.port", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Image", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ProcessID", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ProcessGuid", "target_field": "process.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.CommandLine", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.CurrentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Description", "target_field": "process.pe.description", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Product", "target_field": "process.pe.product", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.OriginalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.FileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Protocol", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.User", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourceHostname", "target_field": "source.hostname", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf new file mode 100644 index 000000000..a5c1d491c --- /dev/null +++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf @@ -0,0 +1,6 @@ +input { + beats { + port => "5044" + tags => [ "beat-ext" ] + } +} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index dcfefa852..e50c04eee 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,22 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} {%- endif %} -# Author: Wes Lambert -# Last Update: 09/14/2018 -filter { - if "beat" in [tags] { - mutate { - ##add_tag => [ "conf_file_9500"] - } - } -} + output { - if "beat" in [tags] { + if "beat-ext" in [tags] { elasticsearch { + pipeline => "beats.common" hosts => "{{ ES }}" index => "so-beats-%{+YYYY.MM.dd}" - template_name => "so-beats" - template => "/so-beats-template.json" + template_name => "so-common" + template => "/so-common-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/templates/so/so-beats-template.json b/salt/logstash/pipelines/templates/so/so-beats-template.json deleted file mode 100644 index 858f1d5e5..000000000 --- a/salt/logstash/pipelines/templates/so/so-beats-template.json +++ /dev/null @@ -1,1286 +0,0 @@ -{ - "index_patterns": [ - "so-beats-*" - ], - "mappings": { - "_meta": { - "version": "6.1.3" - }, - "date_detection": false, - "dynamic_templates": [ - { - "fields": { - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "fields.*" - } - }, - { - "docker.container.labels": { - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "docker.container.labels.*" - } - }, - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "event_data": { - "type":"object", - "dynamic": true - }, - "beat_host": { - "type":"object", - "dynamic": true - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "beat": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "username":{ - "type":"text", - "fields": { - "keyword":{ - "type":"keyword" - } - } - }, - "computer_name": { - "type": "text", - "fields":{ - "keyword":{ - "type":"keyword" - } - } - }, - "docker": { - "properties": { - "container": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "ignore_above": 1024, - "type": "keyword" - }, - "labels": { - "type": "object" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "error": { - "properties": { - "code": { - "type": "long" - }, - "message": { - "norms": false, - "type": "text" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event_id": { - "type": "long" - }, - "fields": { - "type": "object" - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "kubernetes": { - "properties": { - "annotations": { - "type": "object" - }, - "container": { - "properties": { - "image": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "labels": { - "type": "object" - }, - "namespace": { - "ignore_above": 1024, - "type": "keyword" - }, - "pod": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "log_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "message_error": { - "ignore_above": 1024, - "type": "keyword" - }, - "meta": { - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "instance_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "instance_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "project_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "process_id": { - "type": "long" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "record_number": { - "ignore_above": 1024, - "type": "keyword" - }, - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "source_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "thread_id": { - "type": "long" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "domain": { - "type": "keyword" - }, - "identifier": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "user_data": { - "type": "object", - "dynamic": "true" - }, - "version": { - "type": "keyword" - }, - "xml": { - "norms": false, - "type": "text" - }, - "apache2": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "client": { - "ignore_above": 1024, - "type": "keyword" - }, - "code": { - "type": "long" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "auditd": { - "properties": { - "log": { - "properties": { - "a0": { - "ignore_above": 1024, - "type": "keyword" - }, - "acct": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "item": { - "ignore_above": 1024, - "type": "keyword" - }, - "items": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ppid": { - "ignore_above": 1024, - "type": "keyword" - }, - "record_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "res": { - "ignore_above": 1024, - "type": "keyword" - }, - "sequence": { - "type": "long" - } - } - } - } - }, - "fileset": { - "properties": { - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "icinga": { - "properties": { - "debug": { - "properties": { - "facility": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "severity": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "main": { - "properties": { - "facility": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "severity": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "startup": { - "properties": { - "facility": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "severity": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "kafka": { - "properties": { - "log": { - "properties": { - "class": { - "norms": false, - "type": "text" - }, - "component": { - "ignore_above": 1024, - "type": "keyword" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "trace": { - "properties": { - "class": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "norms": false, - "type": "text" - }, - "message": { - "norms": false, - "type": "text" - } - } - } - } - } - } - }, - "logstash": { - "properties": { - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "log_event": { - "type": "object" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "thread": { - "norms": false, - "type": "text" - } - } - }, - "slowlog": { - "properties": { - "event": { - "norms": false, - "type": "text" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "plugin_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "plugin_params": { - "norms": false, - "type": "text" - }, - "plugin_params_object": { - "type": "object" - }, - "plugin_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "thread": { - "norms": false, - "type": "text" - }, - "took_in_millis": { - "type": "long" - }, - "took_in_nanos": { - "type": "long" - } - } - } - } - }, - "mysql": { - "properties": { - "error": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "thread_id": { - "type": "long" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "slowlog": { - "properties": { - "host": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "type": "long" - }, - "ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "lock_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "query": { - "ignore_above": 1024, - "type": "keyword" - }, - "query_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "rows_examined": { - "type": "long" - }, - "rows_sent": { - "type": "long" - }, - "timestamp": { - "type": "long" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "nginx": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "connection_id": { - "type": "long" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - } - } - } - } - }, - "offset": { - "type": "long" - }, - "postgresql": { - "properties": { - "log": { - "properties": { - "database": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "type": "float" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "query": { - "ignore_above": 1024, - "type": "keyword" - }, - "thread_id": { - "type": "long" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "prospector": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "read_timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "redis": { - "properties": { - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "role": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "slowlog": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "cmd": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "properties": { - "us": { - "type": "long" - } - } - }, - "id": { - "type": "long" - }, - "key": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "stream": { - "ignore_above": 1024, - "type": "keyword" - }, - "system": { - "properties": { - "auth": { - "properties": { - "groupadd": { - "properties": { - "gid": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "ssh": { - "properties": { - "dropped_ip": { - "type": "ip" - }, - "event": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - }, - "signature": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "sudo": { - "properties": { - "command": { - "ignore_above": 1024, - "type": "keyword" - }, - "error": { - "ignore_above": 1024, - "type": "keyword" - }, - "pwd": { - "ignore_above": 1024, - "type": "keyword" - }, - "tty": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "useradd": { - "properties": { - "gid": { - "type": "long" - }, - "home": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "shell": { - "ignore_above": 1024, - "type": "keyword" - }, - "uid": { - "type": "long" - } - } - } - } - }, - "syslog": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "traefik": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "backend_url": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "frontend_name": { - "norms": false, - "type": "text" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "request_count": { - "type": "long" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } - }, - "order": 1, - "settings": { - "index": { - "mapping": { - "total_fields": { - "limit": 10000 - } - }, - "number_of_replicas": 0, - "number_of_shards": 1, - "refresh_interval": "30s" - } - } -} From 33f21c05f79937556aa623aaf2a062d539c00f32 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 6 Jun 2020 10:03:25 -0400 Subject: [PATCH 18/30] Fleet standaline custom fqdn fixes --- salt/fleet/event_gen-packages.sls | 4 ++-- setup/so-setup | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/fleet/event_gen-packages.sls b/salt/fleet/event_gen-packages.sls index 72c02be45..6d1e2264a 100644 --- a/salt/fleet/event_gen-packages.sls +++ b/salt/fleet/event_gen-packages.sls @@ -5,9 +5,9 @@ {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} {% if CUSTOM_FLEET_HOSTNAME != None %} - {% set HOSTNAME = {{ CUSTOM_FLEET_HOSTNAME }} %} + {% set HOSTNAME = CUSTOM_FLEET_HOSTNAME %} {% else %} - {% set HOSTNAME = {{ grains.host }} %} + {% set HOSTNAME = grains.host %} {% endif %} so/fleet: diff --git a/setup/so-setup b/setup/so-setup index 7f55c1e46..fbae5b15d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -499,7 +499,7 @@ fi if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then set_progress_str 73 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')" - pillar_override="{"static":{"fleet_custom_hostname":"$FLEETCUSTOMHOSTNAME"}}" + pillar_override=='"{"static":{"fleet_custom_hostname":"'$FLEETCUSTOMHOSTNAME'"}}"' salt-call state.apply -l info fleet.event_update-custom-hostname pillar=$pillar_override >> $setup_log 2>&1 fi From ab6e36e00a3adf0cb0085d436aeec531a968850e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 6 Jun 2020 13:15:44 -0400 Subject: [PATCH 19/30] Fleet syntax error fix --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index fbae5b15d..ea9084831 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -499,7 +499,7 @@ fi if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then set_progress_str 73 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')" - pillar_override=='"{"static":{"fleet_custom_hostname":"'$FLEETCUSTOMHOSTNAME'"}}"' + pillar_override='"{"static":{"fleet_custom_hostname":"'$FLEETCUSTOMHOSTNAME'"}}"' salt-call state.apply -l info fleet.event_update-custom-hostname pillar=$pillar_override >> $setup_log 2>&1 fi From 950d4077f6148b51c0adaa960535fd7d907fe02e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 8 Jun 2020 08:12:14 -0400 Subject: [PATCH 20/30] Update so-setup --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index ea9084831..92147f8f8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -499,7 +499,7 @@ fi if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then set_progress_str 73 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')" - pillar_override='"{"static":{"fleet_custom_hostname":"'$FLEETCUSTOMHOSTNAME'"}}"' + pillar_override=$'\'{"static":{"fleet_custom_hostname":"'$FLEETCUSTOMHOSTNAME$'"}}\'' salt-call state.apply -l info fleet.event_update-custom-hostname pillar=$pillar_override >> $setup_log 2>&1 fi From 51f97f5ae5e1e517c3deeb3bf71a0a7baa5566df Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 8 Jun 2020 08:47:06 -0400 Subject: [PATCH 21/30] Fleet setup - final fixes --- salt/fleet/event_gen-packages.sls | 2 +- setup/so-setup | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/fleet/event_gen-packages.sls b/salt/fleet/event_gen-packages.sls index 6d1e2264a..1bdccea48 100644 --- a/salt/fleet/event_gen-packages.sls +++ b/salt/fleet/event_gen-packages.sls @@ -4,7 +4,7 @@ {% set VERSION = salt['pillar.get']('static:soversion') %} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} -{% if CUSTOM_FLEET_HOSTNAME != None %} +{% if CUSTOM_FLEET_HOSTNAME != None && CUSTOM_FLEET_HOSTNAME != '' %} {% set HOSTNAME = CUSTOM_FLEET_HOSTNAME %} {% else %} {% set HOSTNAME = grains.host %} diff --git a/setup/so-setup b/setup/so-setup index 8e72a1b42..05aa0de27 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -526,8 +526,8 @@ fi if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then set_progress_str 73 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')" - pillar_override=$'\'{"static":{"fleet_custom_hostname":"'$FLEETCUSTOMHOSTNAME$'"}}\'' - salt-call state.apply -l info fleet.event_update-custom-hostname pillar=$pillar_override >> $setup_log 2>&1 + pillar_override="{\"static\":{\"fleet_custom_hostname\": \"$FLEETCUSTOMHOSTNAME\"}}" + salt-call state.apply -l info fleet.event_update-custom-hostname pillar="$pillar_override" >> $setup_log 2>&1 fi set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')" From e2c3fe33428790d7bd32ede568ee8b094b294888 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 8 Jun 2020 12:12:25 -0400 Subject: [PATCH 22/30] [fix] Add X-Forwarded-Proto to nginx configs --- salt/nginx/etc/nginx.conf.so-eval | 24 ++++++++++++++--------- salt/nginx/etc/nginx.conf.so-fleet | 2 +- salt/nginx/etc/nginx.conf.so-master | 24 ++++++++++++++--------- salt/nginx/etc/nginx.conf.so-mastersearch | 24 ++++++++++++++--------- salt/nginx/etc/nginx.conf.so-standalone | 24 ++++++++++++++--------- 5 files changed, 61 insertions(+), 37 deletions(-) diff --git a/salt/nginx/etc/nginx.conf.so-eval b/salt/nginx/etc/nginx.conf.so-eval index 336d27343..579e5730f 100644 --- a/salt/nginx/etc/nginx.conf.so-eval +++ b/salt/nginx/etc/nginx.conf.so-eval @@ -119,6 +119,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location / { @@ -132,6 +133,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location ~ ^/auth/.*?(whoami|login|logout) { @@ -143,7 +145,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef/ { @@ -154,6 +156,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef { @@ -169,6 +172,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /grafana/ { @@ -180,7 +184,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/ { @@ -193,7 +197,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /nodered/ { @@ -206,7 +210,7 @@ http { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /playbook/ { @@ -217,7 +221,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } @@ -230,7 +234,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } {%- if FLEET_NODE %} @@ -246,6 +250,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } {%- endif %} @@ -258,7 +263,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cortex/ { @@ -270,7 +275,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /soctopus/ { @@ -281,7 +286,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/app/soc/ { @@ -304,6 +309,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } error_page 401 = @error401; diff --git a/salt/nginx/etc/nginx.conf.so-fleet b/salt/nginx/etc/nginx.conf.so-fleet index 28372f448..2374af4f9 100644 --- a/salt/nginx/etc/nginx.conf.so-fleet +++ b/salt/nginx/etc/nginx.conf.so-fleet @@ -83,7 +83,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } #error_page 404 /404.html; diff --git a/salt/nginx/etc/nginx.conf.so-master b/salt/nginx/etc/nginx.conf.so-master index 33edb9c3e..4eaca57e7 100644 --- a/salt/nginx/etc/nginx.conf.so-master +++ b/salt/nginx/etc/nginx.conf.so-master @@ -119,6 +119,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location / { @@ -132,6 +133,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location ~ ^/auth/.*?(whoami|login|logout) { @@ -143,7 +145,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef/ { @@ -154,6 +156,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef { @@ -169,6 +172,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /grafana/ { @@ -180,7 +184,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/ { @@ -193,7 +197,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /nodered/ { @@ -206,7 +210,7 @@ http { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /playbook/ { @@ -217,7 +221,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } @@ -230,7 +234,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } {%- if FLEET_NODE %} @@ -246,6 +250,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } {%- endif %} @@ -258,7 +263,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cortex/ { @@ -270,7 +275,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /soctopus/ { @@ -281,7 +286,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/app/soc/ { @@ -304,6 +309,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } error_page 401 = @error401; diff --git a/salt/nginx/etc/nginx.conf.so-mastersearch b/salt/nginx/etc/nginx.conf.so-mastersearch index 33edb9c3e..4eaca57e7 100644 --- a/salt/nginx/etc/nginx.conf.so-mastersearch +++ b/salt/nginx/etc/nginx.conf.so-mastersearch @@ -119,6 +119,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location / { @@ -132,6 +133,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location ~ ^/auth/.*?(whoami|login|logout) { @@ -143,7 +145,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef/ { @@ -154,6 +156,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef { @@ -169,6 +172,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /grafana/ { @@ -180,7 +184,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/ { @@ -193,7 +197,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /nodered/ { @@ -206,7 +210,7 @@ http { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /playbook/ { @@ -217,7 +221,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } @@ -230,7 +234,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } {%- if FLEET_NODE %} @@ -246,6 +250,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } {%- endif %} @@ -258,7 +263,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cortex/ { @@ -270,7 +275,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /soctopus/ { @@ -281,7 +286,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/app/soc/ { @@ -304,6 +309,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } error_page 401 = @error401; diff --git a/salt/nginx/etc/nginx.conf.so-standalone b/salt/nginx/etc/nginx.conf.so-standalone index 33edb9c3e..4eaca57e7 100644 --- a/salt/nginx/etc/nginx.conf.so-standalone +++ b/salt/nginx/etc/nginx.conf.so-standalone @@ -119,6 +119,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location / { @@ -132,6 +133,7 @@ http { proxy_set_header Proxy ""; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; } location ~ ^/auth/.*?(whoami|login|logout) { @@ -143,7 +145,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef/ { @@ -154,6 +156,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /cyberchef { @@ -169,6 +172,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } location /grafana/ { @@ -180,7 +184,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/ { @@ -193,7 +197,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /nodered/ { @@ -206,7 +210,7 @@ http { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /playbook/ { @@ -217,7 +221,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } @@ -230,7 +234,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } {%- if FLEET_NODE %} @@ -246,6 +250,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } {%- endif %} @@ -258,7 +263,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /cortex/ { @@ -270,7 +275,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /soctopus/ { @@ -281,7 +286,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - + proxy_set_header X-Forwarded-Proto $scheme; } location /kibana/app/soc/ { @@ -304,6 +309,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; } error_page 401 = @error401; From 913c1a89bfa85d5cd8d4054283a0c1982ef81369 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 8 Jun 2020 13:28:27 -0400 Subject: [PATCH 23/30] Fleet setup fixes --- salt/fleet/event_gen-packages.sls | 2 +- salt/reactor/fleet.sls | 8 ++++---- setup/so-setup | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/fleet/event_gen-packages.sls b/salt/fleet/event_gen-packages.sls index 1bdccea48..e353eaf92 100644 --- a/salt/fleet/event_gen-packages.sls +++ b/salt/fleet/event_gen-packages.sls @@ -4,7 +4,7 @@ {% set VERSION = salt['pillar.get']('static:soversion') %} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} -{% if CUSTOM_FLEET_HOSTNAME != None && CUSTOM_FLEET_HOSTNAME != '' %} +{% if CUSTOM_FLEET_HOSTNAME != None and CUSTOM_FLEET_HOSTNAME != '' %} {% set HOSTNAME = CUSTOM_FLEET_HOSTNAME %} {% else %} {% set HOSTNAME = grains.host %} diff --git a/salt/reactor/fleet.sls b/salt/reactor/fleet.sls index 9c8023a71..609806de8 100644 --- a/salt/reactor/fleet.sls +++ b/salt/reactor/fleet.sls @@ -9,9 +9,9 @@ import subprocess def run(): MINIONID = data['id'] ACTION = data['data']['action'] - local_salt_dir = /opt/so/saltstack/local - STATICFILE = local_salt_dir + '/pillar/static.sls' - SECRETSFILE = local_salt_dir + '/pillar/secrets.sls' + LOCAL_SALT_DIR = "/opt/so/saltstack/local" + STATICFILE = f"{LOCAL_SALT_DIR}/pillar/static.sls" + SECRETSFILE = f"{LOCAL_SALT_DIR}/pillar/secrets.sls" if MINIONID.split('_')[-1] in ['master','eval','fleet','mastersearch','standalone']: if ACTION == 'enablefleet': @@ -58,7 +58,7 @@ def run(): PACKAGEVERSION += 1 # Run Docker container that will build the packages - gen_packages = subprocess.run(["docker", "run","--rm", "--mount", "type=bind,ssource=" + local_salt_dir + "/salt/fleet/packages,target=/output", \ + gen_packages = subprocess.run(["docker", "run","--rm", "--mount", f"type=bind,source={LOCAL_SALT_DIR}/salt/fleet/packages,target=/output", \ "--mount", "type=bind,source=/etc/ssl/certs/intca.crt,target=/var/launcher/launcher.crt", f"{ MASTER }:5000/soshybridhunter/so-fleet-launcher:{ VERSION }", \ f"{ESECRET}", f"{PACKAGEHOSTNAME}:8090", f"{PACKAGEVERSION}.1.1"], stdout=subprocess.PIPE, encoding='ascii') diff --git a/setup/so-setup b/setup/so-setup index 05aa0de27..c09b5ae7d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -352,7 +352,7 @@ if [[ $is_minion ]]; then copy_ssh_key >> $setup_log 2>&1 fi -if [[ $is_fleet_standalone ]]; then +if [[ "$OSQUERY" = 1 ]]; then host_pillar >> $setup_log 2>&1 fi From e5a23c22fd9bf4756b6b4fcff4773e52739ae180 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 9 Jun 2020 12:16:39 -0400 Subject: [PATCH 24/30] thehive setup fix --- setup/so-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index c09b5ae7d..478151def 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -541,8 +541,8 @@ fi fi if [[ "$THEHIVE" = 1 ]]; then - set_progress_str 76 "$(print_salt_state_apply 'hive')" - salt-call state.apply -l info hive >> $setup_log 2>&1 + set_progress_str 76 "$(print_salt_state_apply 'thehive')" + salt-call state.apply -l info thehive >> $setup_log 2>&1 fi if [[ "$STRELKA" = 1 ]]; then From d09e2b646892b235b22dbb5f4465d99057fb063f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 9 Jun 2020 13:16:03 -0400 Subject: [PATCH 25/30] Dump salt mine to log after applying SSL/CA salt states --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index fda0398a6..ab6d3ba55 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1241,6 +1241,8 @@ salt_checkin() { salt '*' mine.send x509.get_pem_entries glob_path=/etc/pki/ca.crt; echo " Applying SSL state"; salt-call state.apply ssl; + echo " Confirming mine contents" + salt \* mine.get \* x509.get_pem_entries } >> "$setup_log" 2>&1 ;; *) From 1933c267977f02bf8921c0367eddabc3ebaf5fd7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 10 Jun 2020 09:51:56 -0400 Subject: [PATCH 26/30] Additional diagnostic logging for the curious case of the disappearing certificate --- setup/so-functions | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ab6d3ba55..70c96f794 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1237,12 +1237,14 @@ salt_checkin() { sleep 5; systemctl restart salt-minion; sleep 15; + echo " Confirming existence of the CA certificate" + cat /etc/pki/ca.crt echo " Applyng a mine hack"; salt '*' mine.send x509.get_pem_entries glob_path=/etc/pki/ca.crt; + echo " Confirming salt mine now contain the certificate" + salt \* mine.get \* x509.get_pem_entries echo " Applying SSL state"; salt-call state.apply ssl; - echo " Confirming mine contents" - salt \* mine.get \* x509.get_pem_entries } >> "$setup_log" 2>&1 ;; *) From 33105c10aabded6329c40f9c5ecc17a4e295fab3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 10 Jun 2020 14:19:24 -0400 Subject: [PATCH 27/30] [feat] Make automated webuser pass follow requirements --- setup/automation/pm_standalone_defaults | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/pm_standalone_defaults index ae4554a3f..2f62fab98 100644 --- a/setup/automation/pm_standalone_defaults +++ b/setup/automation/pm_standalone_defaults @@ -74,5 +74,5 @@ STRELKA=1 THEHIVE=1 WAZUH=1 WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=onionuser -WEBPASSWD2=onionuser \ No newline at end of file +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r From 05d890901fe123810553aed0adf3e9b49d6247ca Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 11 Jun 2020 07:56:20 -0400 Subject: [PATCH 28/30] Fleet gen packages hostname fix --- salt/fleet/install_package.sls | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/fleet/install_package.sls b/salt/fleet/install_package.sls index 583970bcd..579a2fb32 100644 --- a/salt/fleet/install_package.sls +++ b/salt/fleet/install_package.sls @@ -2,14 +2,24 @@ {%- set FLEETNODE = salt['pillar.get']('static:fleet_node', False) -%} {%- set FLEETHOSTNAME = salt['pillar.get']('static:fleet_hostname', False) -%} {%- set FLEETIP = salt['pillar.get']('static:fleet_ip', False) -%} +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} -{%- if FLEETMASTER or FLEETNODE %} +{% if CUSTOM_FLEET_HOSTNAME != None or CUSTOM_FLEET_HOSTNAME != '' %} + +{{ CUSTOM_FLEET_HOSTNAME }}: + host.present: + - ip: {{ FLEETIP }} + - clean: True + +{% elif FLEETNODE and grains['role'] != 'so-fleet' %} {{ FLEETHOSTNAME }}: host.present: - ip: {{ FLEETIP }} - clean: True +{% endif %} + launcherpkg: pkg.installed: - sources: @@ -18,4 +28,3 @@ launcherpkg: {% elif grains['os'] == 'Ubuntu' %} - launcher-final: salt://fleet/packages/launcher.deb {% endif %} -{%- endif %} From b26a7541cd2243bffff25effb519ddbc3909bbee Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 11 Jun 2020 09:25:09 -0400 Subject: [PATCH 29/30] fix: Force ossec group on /opt/so/wazuh dir tree to eliminate permission failures --- salt/wazuh/init.sls | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index c483f07a0..9004d92ab 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -37,12 +37,6 @@ ossec: - allow_uid_change: True - allow_gid_change: True -#wazuhdir: -# file.directory: -# - name: /opt/so/conf/wazuh -# - user: 945 -# - group: 945 - wazuhpkgs: pkg.installed: - skip_suggestions: False @@ -51,6 +45,13 @@ wazuhpkgs: - hold: True - update_holds: True +wazuhdir: + file.directory: + - name: /opt/so/wazuh + - group: 945 + - recurse: + - group + # Add Wazuh agent conf wazuhagentconf: file.managed: From ab143b321639e4ad5688906c6555b7a93adc1e6b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 11 Jun 2020 11:44:56 -0400 Subject: [PATCH 30/30] Fleet gen packages hostname fix2 --- salt/fleet/install_package.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fleet/install_package.sls b/salt/fleet/install_package.sls index 579a2fb32..3787d6111 100644 --- a/salt/fleet/install_package.sls +++ b/salt/fleet/install_package.sls @@ -4,7 +4,7 @@ {%- set FLEETIP = salt['pillar.get']('static:fleet_ip', False) -%} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} -{% if CUSTOM_FLEET_HOSTNAME != None or CUSTOM_FLEET_HOSTNAME != '' %} +{% if CUSTOM_FLEET_HOSTNAME != (None and '') %} {{ CUSTOM_FLEET_HOSTNAME }}: host.present: