diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index 1aa445c97..6f3ba495b 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -3,7 +3,6 @@ logstash: pipelines: manager: config: - - so/0008_input_fleet_livequery.conf.jinja - so/0009_input_beats.conf - so/0010_input_hhbeats.conf - so/9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 8ae84fe1f..2da8e6c59 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -7,8 +7,7 @@ logstash: - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja - - so/9100_output_osquery.conf.jinja - - so/9101_output_osquery_livequery.conf.jinja + - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 25dfc2fab..0bc2c65b0 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -86,6 +86,19 @@ add_interface_bond0() { fi } +check_airgap() { + # See if this is an airgap install + AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') + if [[ "$AIRGAP" == "True" ]]; then + is_airgap=0 + UPDATE_DIR=/tmp/soagupdate/SecurityOnion + AGDOCKER=/tmp/soagupdate/docker + AGREPO=/tmp/soagupdate/Packages + else + is_airgap=1 + fi +} + check_container() { docker ps | grep "$1:" > /dev/null 2>&1 return $? @@ -97,6 +110,46 @@ check_password() { return $? } +check_elastic_license() { + + [ -n "$TESTING" ] && return + + # See if the user has already accepted the license + if [ ! -f /opt/so/state/yeselastic.txt ]; then + elastic_license + else + echo "Elastic License has already been accepted" + fi +} + +elastic_license() { + +read -r -d '' message <<- EOM +\n +Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License: +https://securityonion.net/elastic-license + +Please review the Elastic License: +https://www.elastic.co/licensing/elastic-license + +Do you agree to the terms of the Elastic License? + +If so, type AGREE to accept the Elastic License and continue. Otherwise, press Enter to exit this program without making any changes. +EOM + +AGREED=$(whiptail --title "Security Onion Setup" --inputbox \ +"$message" 20 75 3>&1 1>&2 2>&3) + +if [ "$AGREED" = 'AGREE' ]; then + mkdir -p /opt/so/state + touch /opt/so/state/yeselastic.txt +else + echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion." + exit 1 +fi + +} + fail() { msg=$1 echo "ERROR: $msg" @@ -250,6 +303,12 @@ set_minionid() { MINIONID=$(lookup_grain id) } +set_palette() { + if [ "$OS" == ubuntu ]; then + update-alternatives --set newt-palette /etc/newt/palette.original + fi +} + set_version() { CURRENTVERSION=0.0.0 if [ -f /etc/soversion ]; then diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 08fbd4628..84627a4ea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -19,13 +19,12 @@ UPDATE_DIR=/tmp/sogh/securityonion INSTALLEDVERSION=$(cat /etc/soversion) +POSTVERSION=$INSTALLEDVERSION INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 SOUP_LOG=/root/soup.log -exec 3>&1 1>${SOUP_LOG} 2>&1 - add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -101,19 +100,6 @@ update_registry() { salt-call state.apply registry queue=True } -check_airgap() { - # See if this is an airgap install - AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') - if [[ "$AIRGAP" == "True" ]]; then - is_airgap=0 - UPDATE_DIR=/tmp/soagupdate/SecurityOnion - AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/Packages - else - is_airgap=1 - fi -} - check_sudoers() { if grep -q "so-setup" /etc/sudoers; then echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." @@ -243,27 +229,35 @@ masterunlock() { fi } -playbook() { - echo "Applying playbook settings" - if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then - salt-call state.apply playbook.OLD_db_init - rm -f /opt/so/rules/elastalert/playbook/*.yaml - so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 & - fi - if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then - so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 & - fi +preupgrade_changes() { + # This function is to add any new pillar items if needed. + echo "Checking to see if changes are needed." + + [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 + [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 + [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 + [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 + [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 } -pillar_changes() { - # This function is to add any new pillar items if needed. - echo "Checking to see if pillar changes are needed." - - [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 - [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 - [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 - [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 - [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 +postupgrade_changes() { + # This function is to add any new pillar items if needed. + echo "Running post upgrade processes." + + [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2 + [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 +} + +post_rc1_to_2.3.21() { + salt-call state.apply playbook.OLD_db_init + rm -f /opt/so/rules/elastalert/playbook/*.yaml + so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 & + POSTVERSION=2.3.21 +} + +post_2.3.2X_to_2.3.30() { + so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 & + POSTVERSION=2.3.30 } rc1_to_rc2() { @@ -408,16 +402,26 @@ up_2.3.2X_to_2.3.30() { check_log_size_limit } -space_check() { - # Check to see if there is enough space +verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then - echo "You are low on disk space. Upgrade will try and clean up space."; - clean_dockers + echo "You are low on disk space." + return 1 else - echo "Plenty of space for upgrading" + return 0 fi - +} + +upgrade_space() { + if ! verify_upgradespace; then + clean_dockers + if ! verify_upgradespace; then + echo "There is not enough space to perform the upgrade. Please free up space and try again" + exit 1 + fi + else + echo "You have enough space for upgrade. Proceeding with soup." + fi } thehive_maint() { @@ -546,6 +550,7 @@ verify_latest_update_script() { } main () { +echo "### Preparing soup at `date` ###" while getopts ":b" opt; do case "$opt" in b ) # process option b @@ -572,6 +577,8 @@ check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" set_os +set_palette +check_elastic_license echo "" if [ $is_airgap -eq 0 ]; then # Let's mount the ISO since this is airgap @@ -598,7 +605,7 @@ fi echo "Let's see if we need to update Security Onion." upgrade_check -space_check +upgrade_space echo "Checking for Salt Master and Minion updates." upgrade_check_salt @@ -648,8 +655,7 @@ else echo "" fi -echo "Making pillar changes." -pillar_changes +preupgrade_changes echo "" if [ $is_airgap -eq 0 ]; then @@ -703,7 +709,7 @@ echo "Starting Salt Master service." systemctl start salt-master echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True -playbook +postupgrade_changes unmount_update thehive_maint @@ -740,6 +746,7 @@ NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) if [ $NUM_MINIONS -gt 1 ]; then cat << EOF + This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch. Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete. @@ -750,7 +757,23 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html EOF fi - +echo "### soup has been served at `date` ###" } -main "$@" | tee /dev/fd/3 +cat << EOF + +SOUP - Security Onion UPdater + +Please review the following for more information about the update process and recent updates: +https://docs.securityonion.net/soup +https://blog.securityonion.net + +Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup. + +Press Enter to continue or Ctrl-C to cancel. + +EOF + +read input + +main "$@" | tee -a $SOUP_LOG diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index 63eb6077a..e8e462053 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -32,8 +32,6 @@ { "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, { "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, - { "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, - { "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 012c590d0..ebf123fed 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -51,16 +51,29 @@ "match_mapping_type": "string", "path_match": "*.ip", "mapping": { - "type": "ip" + "type": "ip", + "fields" : { + "keyword" : { + "ignore_above" : 45, + "type" : "keyword" + } + } + } } }, { "port": { - "match_mapping_type": "string", "path_match": "*.port", "mapping": { - "type": "integer" + "type": "integer", + "fields" : { + "keyword" : { + "ignore_above" : 6, + "type" : "keyword" + } + } + } } }, diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index c9618554d..1d6ba350f 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -26,15 +26,6 @@ iptables_fix_fwd: - position: 1 - target: DOCKER-USER -# Allow related/established sessions -iptables_allow_established: - iptables.append: - - table: filter - - chain: INPUT - - jump: ACCEPT - - match: conntrack - - ctstate: 'RELATED,ESTABLISHED' - # I like pings iptables_allow_pings: iptables.append: @@ -77,17 +68,6 @@ enable_docker_user_fw_policy: - out-interface: docker0 - position: 1 -enable_docker_user_established: - iptables.insert: - - table: filter - - chain: DOCKER-USER - - jump: ACCEPT - - in-interface: '!docker0' - - out-interface: docker0 - - position: 1 - - match: conntrack - - ctstate: 'RELATED,ESTABLISHED' - {% set count = namespace(value=0) %} {% for chain, hg in assigned_hostgroups.chain.items() %} {% for hostgroup, portgroups in assigned_hostgroups.chain[chain].hostgroups.items() %} @@ -120,6 +100,27 @@ enable_docker_user_established: {% endfor %} {% endfor %} +# Allow related/established sessions +iptables_allow_established: + iptables.insert: + - table: filter + - chain: INPUT + - jump: ACCEPT + - position: 1 + - match: conntrack + - ctstate: 'RELATED,ESTABLISHED' + +enable_docker_user_established: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - in-interface: '!docker0' + - out-interface: docker0 + - position: 1 + - match: conntrack + - ctstate: 'RELATED,ESTABLISHED' + # Block icmp timestamp response block_icmp_timestamp_reply: iptables.append: diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index c857b2857..0ee2508eb 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -24,3 +24,9 @@ THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7} # Load saved objects curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1 + +# Disable certain Features from showing up in the Kibana UI + +echo +echo "Setting up default Space:" +curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' | jq \ No newline at end of file diff --git a/salt/strelka/files/filestream/filestream.yaml b/salt/strelka/files/filestream/filestream.yaml index aa5d51ad1..57ef65127 100644 --- a/salt/strelka/files/filestream/filestream.yaml +++ b/salt/strelka/files/filestream/filestream.yaml @@ -19,7 +19,8 @@ files: - '/nsm/strelka/unprocessed/*' delete: false gatekeeper: true + processed: '/nsm/strelka/processed' response: report: 5s delta: 5s -staging: '/nsm/strelka/processed' +staging: '/nsm/strelka/staging' diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 91226701d..d90484fed 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -86,6 +86,13 @@ strelkaprocessed: - group: 939 - makedirs: True +strelkastaging: + file.directory: + - name: /nsm/strelka/staging + - user: 939 + - group: 939 + - makedirs: True + strelkaunprocessed: file.directory: - name: /nsm/strelka/unprocessed @@ -96,7 +103,7 @@ strelkaunprocessed: # Check to see if Strelka frontend port is available strelkaportavailable: cmd.run: - - name: netstat -utanp | grep ":57314" | grep -qv docker && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 strelka_coordinator: docker_container.running: @@ -213,4 +220,4 @@ strelka_zeek_extracted_sync: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index b8976b8c9..f6bcbdaf5 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -618,11 +618,8 @@ # # Read stats from one or more Elasticsearch servers or clusters {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] - -# ## specify a list of one or more Elasticsearch servers -# # you can add username and password to your url to use basic authentication: -# # servers = ["http://user:pass@localhost:9200"] servers = ["https://{{ MANAGER }}:9200"] + insecure_skip_verify = true {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] diff --git a/setup/so-functions b/setup/so-functions index 355e48fbe..82da7c98d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1243,8 +1243,13 @@ es_heapsize() { # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html ES_HEAP_SIZE="25000m" else - # Set heap size to 25% of available memory - ES_HEAP_SIZE=$(( total_mem / 4 ))"m" + # Set heap size to 33% of available memory + ES_HEAP_SIZE=$(( total_mem / 3 )) + if [ "$ES_HEAP_SIZE" -ge 25001 ] ; then + ES_HEAP_SIZE="25000m" + else + ES_HEAP_SIZE=$ES_HEAP_SIZE"m" + fi fi export ES_HEAP_SIZE @@ -1686,8 +1691,8 @@ manager_global() { " so-zeek:"\ " shards: 5"\ " warm: 7"\ - " close: 365"\ - " delete: 45"\ + " close: 45"\ + " delete: 365"\ "minio:"\ " access_key: '$ACCESS_KEY'"\ " access_secret: '$ACCESS_SECRET'"\ diff --git a/setup/so-setup b/setup/so-setup index fb4f76be0..de0ffefd4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -168,10 +168,8 @@ local_sbin="$(pwd)/../salt/common/tools/sbin" export PATH=$PATH:$local_sbin set_network_dev_status_list +set_palette >> $setup_log 2>&1 -if [ "$OS" == ubuntu ]; then - update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1 -fi # Kernel messages can overwrite whiptail screen #812 # https://github.com/Security-Onion-Solutions/securityonion/issues/812 @@ -274,6 +272,10 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then fi fi +if [[ $is_manager || $is_import ]]; then + check_elastic_license +fi + if ! [[ -f $install_opt_file ]]; then if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone"