From 968fee3488eee120dcf1fc1e403539e09e93c459 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Aug 2023 13:10:41 -0400 Subject: [PATCH 01/26] Regen Agent Installers when Fleet URLs change --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 6 ++++++ .../tools/sbin_jinja/so-elastic-fleet-urls-update | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 2a19dcbd9..d7d6458c9 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -11,6 +11,12 @@ . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common +LOG="/opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log" + +# Check to see if we are already running +NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers") +[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING gen installers script processes running...exiting." >>$LOG && exit 0 + for i in {1..30} do ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 24c5dabed..4a744665a 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -62,7 +62,7 @@ fi NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') -# Compare the current & new list of URLs - if different, update the Fleet Server URLs +# Compare the current & new list of URLs - if different, update the Fleet Server URLs & regenerate the agent installer if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then printf "\nHashes match - no update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" @@ -71,4 +71,5 @@ else printf "\nHashes don't match - update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" update_fleet_urls + /sbin/so-elastic-agent-gen-installers & fi From b520c1abb777a479df05e7e033edfa7b57b37d77 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 10:36:40 -0400 Subject: [PATCH 02/26] Allow multiple Custom Fleet FQDN --- salt/elasticfleet/defaults.yaml | 3 ++- salt/elasticfleet/soc_elasticfleet.yaml | 2 +- .../sbin_jinja/so-elastic-fleet-outputs-update | 14 +++++++++----- .../sbin_jinja/so-elastic-fleet-urls-update | 16 ++++++++++------ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 3d806d63f..62a1302c1 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,7 +2,8 @@ elasticfleet: enabled: False config: server: - custom_fqdn: '' + custom_fqdn: + - '' enable_auto_configuration: True endpoints_enrollment: '' es_token: '' diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 9b918f0ac..772e68181 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -12,7 +12,7 @@ elasticfleet: config: server: custom_fqdn: - description: Custom FQDN for Agents to connect to. + description: Custom FQDN for Agents to connect to. One per line. global: True helpLink: elastic-fleet.html advanced: True diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 042084d84..400a6224f 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -2,7 +2,6 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} . /usr/sbin/so-common @@ -41,10 +40,15 @@ else NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055") fi -{% if CUSTOMFQDN != "" %} -# Add Custom Hostname to list -NEW_LIST+=("{{ CUSTOMFQDN }}:5055") -{% endif %} +# Query for FQDN entries & add them to the list +CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +if [ -n "$CUSTOMFQDNLIST" ]; then + readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST + for CUSTOMNAME in "${CUSTOMFQDN[@]}" + do + NEW_LIST+=("$CUSTOMNAME:5055") + done +fi # Query for the current Grid Nodes that are running Logstash LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 4a744665a..52727780d 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,7 +2,6 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} . /usr/sbin/so-common @@ -41,10 +40,15 @@ else NEW_LIST=("https://{{ GLOBALS.url_base }}:8220" "https://{{ GLOBALS.hostname }}:8220") fi -{% if CUSTOMFQDN != "" %} -# Add Custom Hostname to list -NEW_LIST+=("https://{{ CUSTOMFQDN }}:8220") -{% endif %} +# Query for FQDN entries & add them to the list +CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +if [ -n "$CUSTOMFQDNLIST" ]; then + readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST + for CUSTOMNAME in "${CUSTOMFQDN[@]}" + do + NEW_LIST+=("https://$CUSTOMNAME:8220") + done +fi # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') @@ -71,5 +75,5 @@ else printf "\nHashes don't match - update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" update_fleet_urls - /sbin/so-elastic-agent-gen-installers & + /sbin/so-elastic-agent-gen-installers >> /opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log & fi From f153c1125d9dba74b5358c298936fbd0b873c2f8 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 15:23:18 -0400 Subject: [PATCH 03/26] Allow multiple Custom Fleet FQDN --- salt/elasticfleet/defaults.yaml | 3 +-- salt/elasticfleet/enabled.sls | 13 ++++++++++--- salt/elasticfleet/soc_elasticfleet.yaml | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 62a1302c1..0ae7a5176 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,8 +2,7 @@ elasticfleet: enabled: False config: server: - custom_fqdn: - - '' + custom_fqdn: [] enable_auto_configuration: True endpoints_enrollment: '' es_token: '' diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 025a87e14..bb6410f2c 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -15,6 +15,7 @@ include: - elasticfleet.config - elasticfleet.sostatus + - ssl # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} @@ -61,11 +62,14 @@ so-elastic-fleet: - {{ BINDING }} {% endfor %} - binds: - - /etc/pki:/etc/pki:ro + - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro + - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro + - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro {% if GLOBALS.os_family == 'Debian' %} - - /etc/ssl:/etc/ssl:ro + - /etc/ssl/elasticfleet-server.crt:/etc/ssl/elasticfleet-server.crt:ro + - /etc/ssl/elasticfleet-server.key:/etc/ssl/elasticfleet-server.key:ro + - /etc/ssl/tls/certs/intca.crt:/etc/ssl/tls/certs/intca.crt:ro {% endif %} - #- /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state:rw - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -93,6 +97,9 @@ so-elastic-fleet: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - x509: etc_elasticfleet_key + - x509: etc_elasticfleet_crt {% endif %} {% if GLOBALS.role != "so-fleet" %} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 772e68181..af660358a 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -16,6 +16,7 @@ elasticfleet: global: True helpLink: elastic-fleet.html advanced: True + forcedType: "[]string" enable_auto_configuration: description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs. global: True From ab28cee7cf3041ac6276b120956e8d117a1323b4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 17:45:37 -0400 Subject: [PATCH 04/26] Allow multiple Custom Fleet FQDN --- salt/ssl/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 97e971b83..15c29791f 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -7,7 +7,7 @@ {% if sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} {% set global_ca_text = [] %} {% set global_ca_server = [] %} @@ -154,7 +154,7 @@ etc_elasticfleet_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -211,7 +211,7 @@ etc_elasticfleet_logstash_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 1c8a8c460c90572cbeea725a88a60dc358c5b5f9 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 17:53:29 -0400 Subject: [PATCH 05/26] Restart logstash when certs change --- salt/logstash/enabled.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index cd9d6dd7e..a33080f8d 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -22,6 +22,7 @@ include: {% endif %} - logstash.config - logstash.sostatus + - ssl so-logstash: docker_container.running: @@ -90,6 +91,8 @@ so-logstash: {% endfor %} {% endif %} - watch: + - x509: etc_elasticfleet_logstash_key + - x509: etc_elasticfleet_logstash_crt - file: lsetcsync {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} - file: ls_pipeline_{{assigned_pipeline}} From 1bc7bbc76efe8dcead09867a34358f64ea71e2c5 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 20:02:37 -0400 Subject: [PATCH 06/26] Refactor custom_fqdn --- salt/ssl/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 15c29791f..063172e00 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -154,7 +154,7 @@ etc_elasticfleet_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -211,7 +211,7 @@ etc_elasticfleet_logstash_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 3368789b43f78c6fa3616151d94de4ee99c46a66 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 08:49:45 -0400 Subject: [PATCH 07/26] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 59aa62c1f..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.5 +2.4.0-foxtrot From 3847863b3d471e69e591e5d1c9c9d26fce569f51 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 08:51:23 -0400 Subject: [PATCH 08/26] Add time shift --- salt/common/tools/sbin_jinja/so-import-evtx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index fec7223b8..5920f58c1 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -44,6 +44,10 @@ while [[ $# -gt 0 ]]; do --quiet) quiet=1 ;; + --shift) + SHIFTDATE=$1 + shift + ;; -*) echo "Encountered unexpected parameter: $param" usage @@ -68,8 +72,10 @@ function status { function evtx2es() { EVTX=$1 HASH=$2 + SHIFTDATE=$3 docker run --rm \ + -e "SHIFTTS=$SHIFTDATE" \ -v "$EVTX:/tmp/data.evtx" \ -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \ -v "/nsm/import/evtx-end_newest:/tmp/newest" \ @@ -113,7 +119,9 @@ echo $END_NEWEST > /nsm/import/evtx-end_newest for EVTX in $INPUT_FILES; do EVTX=$(/usr/bin/realpath "$EVTX") status "Processing Import: ${EVTX}" - + if ! [ -z "$SHIFTDATE" ]; then + status "- timeshifting logs to end date of $SHIFTDATE" + fi # generate a unique hash to assist with dedupe checks HASH=$(md5sum "${EVTX}" | awk '{ print $1 }') HASH_DIR=/nsm/import/${HASH} @@ -136,7 +144,7 @@ for EVTX in $INPUT_FILES; do # import evtx and write them to import ingest pipeline status "- importing logs to Elasticsearch..." - evtx2es "${EVTX}" $HASH + evtx2es "${EVTX}" $HASH "$SHIFTDATE" if [[ $? -ne 0 ]]; then INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1)) status "- WARNING: This evtx file may not have fully imported successfully" @@ -222,4 +230,4 @@ if [[ $json -eq 1 ]]; then }''' fi -exit $RESULT \ No newline at end of file +exit $RESULT From cf2233bbb6702c4e4da396ade6449373493a933b Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 08:54:54 -0400 Subject: [PATCH 09/26] Add help information for time shift --- salt/common/tools/sbin_jinja/so-import-evtx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index 5920f58c1..dff2133cf 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -27,6 +27,8 @@ Imports one or more evtx files into Security Onion. The evtx files will be analy Options: --json Outputs summary in JSON format. Implies --quiet. --quiet Silences progress information to stdout. + --shift Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly. + Ex. sudo so-import-evtx --shift 2023-08-01T01:01:01.00000Z example.evtx EOF } From 80598d7f8d26530f4ce55ea097f6fab526c94131 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 3 Aug 2023 14:36:47 -0400 Subject: [PATCH 10/26] Update soup for airgap --- salt/manager/tools/sbin/soup | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f47c1d5e2..0cea456f4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -184,7 +184,7 @@ check_airgap() { is_airgap=0 UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/Packages + AGREPO=/tmp/soagupdate/minimal/Packages else is_airgap=1 fi @@ -402,9 +402,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3 [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 - [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 - - + [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 true } @@ -479,11 +477,22 @@ up_to_2.4.4() { } up_to_2.4.5() { - update_elastic_agent + determine_elastic_agent_upgrade INSTALLEDVERSION=2.4.5 } +determine_elastic_agent_upgrade() { + if [[ $is_airgap -eq 0 ]]; then + update_elastic_agent_airgap + else + update_elastic_agent +} + +update_elastic_agent_airgap() { + rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/ +} + verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then @@ -521,6 +530,7 @@ update_centos_repo() { echo "Syncing new updates to /nsm/repo" rsync -av $AGREPO/* /nsm/repo/ echo "Creating repo" + dnf -y install yum-utils createrepo createrepo /nsm/repo } From 27b70cbf6891d021981d5c798332f602f8612b25 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 15:21:20 -0400 Subject: [PATCH 11/26] Use jinja instead --- .../tools/sbin_jinja/so-elastic-fleet-urls-update | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 52727780d..c484fa704 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,6 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common @@ -41,7 +42,8 @@ else fi # Query for FQDN entries & add them to the list -CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} +CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) if [ -n "$CUSTOMFQDNLIST" ]; then readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST for CUSTOMNAME in "${CUSTOMFQDN[@]}" @@ -49,6 +51,7 @@ if [ -n "$CUSTOMFQDNLIST" ]; then NEW_LIST+=("https://$CUSTOMNAME:8220") done fi +{% endif %} # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') From e78fcbc6cbc2fa4362e45e378eebfcc80a0d8fc9 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 15:25:11 -0400 Subject: [PATCH 12/26] Refactor for Jinja instead --- .../tools/sbin_jinja/so-elastic-fleet-outputs-update | 7 +++++-- .../tools/sbin_jinja/so-elastic-fleet-urls-update | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 400a6224f..17c867c07 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -2,6 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common @@ -41,14 +42,16 @@ else fi # Query for FQDN entries & add them to the list -CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} +CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) if [ -n "$CUSTOMFQDNLIST" ]; then readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST for CUSTOMNAME in "${CUSTOMFQDN[@]}" do - NEW_LIST+=("$CUSTOMNAME:5055") + NEW_LIST+=("https://$CUSTOMNAME:8220") done fi +{% endif %} # Query for the current Grid Nodes that are running Logstash LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index c484fa704..7d29fe080 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,7 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common From 9319c3f2e1c7757abcd2fc68b6d1a2e7713a0b39 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 3 Aug 2023 15:27:24 -0400 Subject: [PATCH 13/26] Update soup for airgap --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0cea456f4..cede5c438 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -487,6 +487,7 @@ determine_elastic_agent_upgrade() { update_elastic_agent_airgap else update_elastic_agent + fi } update_elastic_agent_airgap() { From 3e4136e641c27dbd0f2a08cd870550c640d65a3f Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 15:56:05 -0400 Subject: [PATCH 14/26] Update help text --- salt/common/tools/sbin_jinja/so-import-evtx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index dff2133cf..59a13612c 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -28,7 +28,7 @@ Options: --json Outputs summary in JSON format. Implies --quiet. --quiet Silences progress information to stdout. --shift Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly. - Ex. sudo so-import-evtx --shift 2023-08-01T01:01:01.00000Z example.evtx + Ex. sudo so-import-evtx --shift "2023-08-01 01:01:01" example.evtx EOF } From ca6276b922c86b0df31a25670b6872a6b6523b5e Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 15:58:33 -0400 Subject: [PATCH 15/26] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..59aa62c1f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.5 From 6b5343f582d5651a1b7ad1bcb403f106796af630 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 16:25:02 -0400 Subject: [PATCH 16/26] Update for 8.8.2 --- .../endpoints-initial/elastic-defend-endpoints.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json index 6ffb6418e..8ab4f748e 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json @@ -9,13 +9,12 @@ }, "enabled": true, "policy_id": "endpoints-initial", - "vars": {}, "inputs": [{ - "type": "endpoint", + "type": "ENDPOINT_INTEGRATION_CONFIG", "enabled": true, "streams": [], "config": { - "integration_config": { + "_config": { "value": { "type": "endpoint", "endpointConfig": { From 2caca92082ef54c71ad617b469ca699bdbcf6418 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 3 Aug 2023 17:11:43 -0400 Subject: [PATCH 17/26] Raid refactor + yara and rule proxy --- salt/common/tools/sbin_jinja/so-raid-status | 116 ++++++++---------- salt/idstools/tools/sbin_jinja/so-rule-update | 16 ++- .../manager/tools/sbin_jinja/so-yara-download | 3 +- 3 files changed, 62 insertions(+), 73 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status index c5ac5fac6..0249f4ccd 100755 --- a/salt/common/tools/sbin_jinja/so-raid-status +++ b/salt/common/tools/sbin_jinja/so-raid-status @@ -1,7 +1,7 @@ #!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -9,25 +9,26 @@ . /usr/sbin/so-common -appliance_check() { - {%- if salt['grains.get']('sosmodel', '') %} - APPLIANCE=1 - {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %} - exit 0 - {%- endif %} - DUDEYOUGOTADELL=$(dmidecode |grep Dell) - if [[ -n $DUDEYOUGOTADELL ]]; then - APPTYPE=dell - else - APPTYPE=sm - fi - mkdir -p /opt/so/log/raid - - {%- else %} - echo "This is not an appliance" - exit 0 - {%- endif %} -} +{%- if salt['grains.get']('sosmodel', '') %} +{%- set model = salt['grains.get']('sosmodel') %} +model={{ model }} +# Don't need cloud images to use this +if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then + exit 0 +fi +{%- else %} +echo "This is not an appliance" +exit 0 +{%- endif %} +if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then + is_bossraid=true +fi +if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then + is_swraid=true +fi +if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then + is_hwraid=true +fi check_nsm_raid() { PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl) @@ -49,61 +50,44 @@ check_nsm_raid() { check_boss_raid() { MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional) - if [[ -n $DUDEYOUGOTADELL ]]; then - if [[ -n $MVCLI ]]; then - BOSSRAID=0 - else - BOSSRAID=1 - fi + if [[ -n $MVCLI ]]; then + BOSSRAID=0 + else + BOSSRAID=1 fi } check_software_raid() { - if [[ -n $DUDEYOUGOTADELL ]]; then - SWRC=$(grep "_" /proc/mdstat) - - if [[ -n $SWRC ]]; then - # RAID is failed in some way - SWRAID=1 - else - SWRAID=0 - fi + SWRC=$(grep "_" /proc/mdstat) + if [[ -n $SWRC ]]; then + # RAID is failed in some way + SWRAID=1 + else + SWRAID=0 fi } -# This script checks raid status if you use SO appliances +# Set everything to 0 +SWRAID=0 +BOSSRAID=0 +HWRAID=0 -# See if this is an appliance - -appliance_check -check_nsm_raid -check_boss_raid -{%- if salt['grains.get']('sosmodel', '') %} -{%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %} -check_software_raid -{%- endif %} -{%- endif %} - -if [[ -n $SWRAID ]]; then - if [[ $SWRAID == '0' && $BOSSRAID == '0' ]]; then - RAIDSTATUS=0 - else - RAIDSTATUS=1 - fi -elif [[ -n $DUDEYOUGOTADELL ]]; then - if [[ $BOSSRAID == '0' && $HWRAID == '0' ]]; then - RAIDSTATUS=0 - else - RAIDSTATUS=1 - fi -elif [[ "$APPTYPE" == 'sm' ]]; then - if [[ -n "$HWRAID" ]]; then - RAIDSTATUS=0 - else - RAIDSTATUS=1 - fi +if [[ $is_hwraid ]]; then + check_nsm_raid +fi +if [[ $is_bossraid ]]; then + check_boss_raid +fi +if [[ $is_swraid ]]; then + check_software_raid fi -echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log +sum=$(($SWRAID + $BOSSRAID + $HWRAID)) +if [[ $sum == "0" ]]; then + RAIDSTATUS=0 +else + RAIDSTATUS=1 +fi +echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log \ No newline at end of file diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update index 3e4b382e6..504831f9f 100755 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ b/salt/idstools/tools/sbin_jinja/so-rule-update @@ -3,17 +3,21 @@ {%- from 'vars/globals.map.jinja' import GLOBALS %} {%- from 'idstools/map.jinja' import IDSTOOLSMERGED %} -{%- set proxy = salt['pillar.get']('manager:proxy') %} + +{%- set proxy = salt['pillar.get']('manager:proxy') %} +{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} + +# Download the rules from the internet +{%- if proxy %} +export http_proxy={{ proxy }} +export https_proxy={{ proxy }} +export no_proxy="{{ noproxy }}" +{%- endif %} mkdir -p /nsm/rules/suricata chown -R socore:socore /nsm/rules/suricata # Download the rules from the internet {%- if GLOBALS.airgap != 'True' %} -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy=salt['pillar.get']('manager:no_proxy') -{%- endif %} {%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force {%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download index e9b991b6c..aa9576253 100644 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ b/salt/manager/tools/sbin_jinja/so-yara-download @@ -3,12 +3,13 @@ NOROOT=1 . /usr/sbin/so-common {%- set proxy = salt['pillar.get']('manager:proxy') %} +{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} # Download the rules from the internet {%- if proxy %} export http_proxy={{ proxy }} export https_proxy={{ proxy }} -export no_proxy=salt['pillar.get']('manager:no_proxy') +export no_proxy="{{ noproxy }}" {%- endif %} repos="/opt/so/conf/strelka/repos.txt" From 2472d6a7279e025e6714925dac83bb7c9f9eca42 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 18:52:29 -0400 Subject: [PATCH 18/26] Don't watch certs on search nodes --- salt/logstash/enabled.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index a33080f8d..731ad4ca3 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -91,8 +91,10 @@ so-logstash: {% endfor %} {% endif %} - watch: + {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-fleet', 'so-receiver'] %} - x509: etc_elasticfleet_logstash_key - x509: etc_elasticfleet_logstash_crt + {% endif %} - file: lsetcsync {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} - file: ls_pipeline_{{assigned_pipeline}} From 593cdbd06001f3492423c1c590b4e5ac3d5ae92d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 08:50:06 -0400 Subject: [PATCH 19/26] add rules for idh to connect to managers, change idh from sensor to idh in so-firewall-minion --- salt/firewall/defaults.yaml | 33 ++++++++++++++++++++++ salt/manager/tools/sbin/so-firewall-minion | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 3095c052e..48074b0be 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -383,6 +383,17 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + idh: + portgroups: + - docker_registry + - influxdb + - sensoroni + - yum + - beats_5044 + - beats_5644 + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update sensor: portgroups: - beats_5044 @@ -548,6 +559,17 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + idh: + portgroups: + - docker_registry + - influxdb + - sensoroni + - yum + - beats_5044 + - beats_5644 + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update sensor: portgroups: - beats_5044 @@ -723,6 +745,17 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + idh: + portgroups: + - docker_registry + - influxdb + - sensoroni + - yum + - beats_5044 + - beats_5644 + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update sensor: portgroups: - docker_registry diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index 4834f0e41..7b0ddab90 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -74,9 +74,9 @@ fi so-firewall includehost heavynode "$IP" --apply ;; 'IDH') - so-firewall includehost sensor "$IP" --apply + so-firewall includehost idh "$IP" --apply ;; 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - esac \ No newline at end of file + esac From 682289ef23736b687cd271a503c58200143c4c9f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:01:09 -0400 Subject: [PATCH 20/26] add sensoroni ports where missing --- salt/firewall/defaults.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 48074b0be..125bf0f08 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -370,6 +370,7 @@ firewall: - elastic_agent_data - elastic_agent_update - localrules + - sensoroni fleet: portgroups: - elasticsearch_rest @@ -404,6 +405,7 @@ firewall: - yum - docker_registry - influxdb + - sensoroni searchnode: portgroups: - redis @@ -416,6 +418,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni heavynode: portgroups: - redis @@ -428,6 +431,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni receiver: portgroups: - yum @@ -436,6 +440,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni beats_endpoint: portgroups: - beats_5044 @@ -546,6 +551,7 @@ firewall: - elastic_agent_data - elastic_agent_update - localrules + - sensoroni fleet: portgroups: - elasticsearch_rest @@ -580,6 +586,7 @@ firewall: - yum - docker_registry - influxdb + - sensoroni searchnode: portgroups: - redis @@ -591,6 +598,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni heavynode: portgroups: - redis @@ -602,6 +610,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni receiver: portgroups: - yum @@ -610,6 +619,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni beats_endpoint: portgroups: - beats_5044 @@ -793,6 +803,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni beats_endpoint: portgroups: - beats_5044 From dd1fa51eb5bb2dc916401af541cc961c531497f2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 4 Aug 2023 09:03:17 -0400 Subject: [PATCH 21/26] Generate community_id for defend endpoint logs --- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 0c317ae48..45583a464 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -78,6 +78,7 @@ { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, + {"community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } ], "on_failure": [ From 78950ebfbb39d21ec3917b9bb3819c420b0935cc Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 4 Aug 2023 09:16:58 -0400 Subject: [PATCH 22/26] Update so-whiptail --- setup/so-whiptail | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 4e9ccea60..c55e2db8f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1012,9 +1012,9 @@ whiptail_manager_unreachable() { local msg read -r -d '' msg <<- EOM - Setup is unable to access the manager at this time. + Setup is unable to access the manager. This most likely means that you need to allow this machine to connect through the manager's firewall. - Run the following on the manager: + You can either go to SOC --> Administration --> Configuration and choose the correct firewall option from the list OR you can run the following command on the manager: sudo so-firewall-minion --role=$install_type --ip=$MAINIP From a51acfc314004e9c2f066fe387a85f34a92ab7da Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:17:22 -0400 Subject: [PATCH 23/26] rename analyst to workstation for fw rules. allow workstation to connect to salt_manager port on managers --- salt/firewall/defaults.yaml | 31 ++++++++++++++-------- salt/firewall/soc_firewall.yaml | 2 +- salt/manager/tools/sbin/so-firewall-minion | 3 +++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 125bf0f08..0d32d57ca 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1,6 +1,5 @@ firewall: hostgroups: - analyst: [] anywhere: - 0.0.0.0/0 beats_endpoint: [] @@ -26,6 +25,7 @@ firewall: standalone: [] strelka_frontend: [] syslog: [] + workstation: [] customhostgroup0: [] customhostgroup1: [] customhostgroup2: [] @@ -215,9 +215,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -458,9 +458,9 @@ firewall: endgame: portgroups: - endgame - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -507,6 +507,9 @@ firewall: receiver: portgroups: - salt_manager + workstation: + portgroups: + - salt_manager self: portgroups: - syslog @@ -637,9 +640,9 @@ firewall: endgame: portgroups: - endgame - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -686,6 +689,9 @@ firewall: receiver: portgroups: - salt_manager + workstation: + portgroups: + - salt_manager self: portgroups: - syslog @@ -824,9 +830,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -876,6 +882,9 @@ firewall: receiver: portgroups: - salt_manager + workstation: + portgroups: + - salt_manager self: portgroups: - syslog @@ -1169,9 +1178,9 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 0011a245e..78c0ebc73 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -1,6 +1,6 @@ firewall: hostgroups: - analyst: &hostgroupsettings + workstation: &hostgroupsettings description: List of IP or CIDR blocks to allow access to this hostgroup. forcedType: "[]string" helplink: firewall.html diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index 7b0ddab90..d3bbb3eeb 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,4 +79,7 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; + 'WORKSTATION') + so-firewall includehost workstation "$IP" --apply + ;; esac From 726ec7235000959622e9af7df4f5a80dc6aa1fb3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:22:59 -0400 Subject: [PATCH 24/26] allow idh to connect to salt_manager ports on managres --- salt/firewall/defaults.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 0d32d57ca..ff776d309 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -492,6 +492,9 @@ firewall: fleet: portgroups: - salt_manager + idh: + portgroups: + - salt_manager localhost: portgroups: - all @@ -674,6 +677,9 @@ firewall: fleet: portgroups: - salt_manager + idh: + portgroups: + - salt_manager localhost: portgroups: - all @@ -863,7 +869,10 @@ firewall: - all fleet: portgroups: - - salt_manager + - salt_manager + idh: + portgroups: + - salt_manager localhost: portgroups: - all From 0f52530d0760cf67cbda82ee81d18b220fe3cc17 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:37:58 -0400 Subject: [PATCH 25/26] soc_firewall.yaml update adding idh and rename analyst to workstation --- salt/firewall/soc_firewall.yaml | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 78c0ebc73..27c52e123 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -213,7 +213,7 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -338,7 +338,9 @@ firewall: DOCKER-USER: hostgroups: manager: - portgroups: *portgroupsdocker + portgroups: *portgroupsdocker + idh: + portgroups: *portgroupsdocker sensor: portgroups: *portgroupsdocker searchnode: @@ -359,7 +361,7 @@ firewall: portgroups: *portgroupsdocker endgame: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -389,12 +391,16 @@ firewall: portgroups: *portgroupshost localhost: portgroups: *portgroupshost + idh: + portgroups: *portgroupshost sensor: portgroups: *portgroupshost searchnode: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost + workstation: + portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost customhostgroup1: @@ -422,6 +428,8 @@ firewall: hostgroups: managersearch: portgroups: *portgroupsdocker + idh: + portgroups: *portgroupsdocker sensor: portgroups: *portgroupsdocker searchnode: @@ -442,7 +450,7 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -472,12 +480,16 @@ firewall: portgroups: *portgroupshost localhost: portgroups: *portgroupshost + idh: + portgroups: *portgroupshost sensor: portgroups: *portgroupshost searchnode: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost + workstation: + portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost customhostgroup1: @@ -509,6 +521,8 @@ firewall: portgroups: *portgroupsdocker fleet: portgroups: *portgroupsdocker + idh: + portgroups: *portgroupsdocker sensor: portgroups: *portgroupsdocker searchnode: @@ -531,7 +545,7 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -565,12 +579,16 @@ firewall: portgroups: *portgroupshost standalone: portgroups: *portgroupshost + idh: + portgroups: *portgroupshost sensor: portgroups: *portgroupshost searchnode: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost + workstation: + portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost customhostgroup1: @@ -793,7 +811,7 @@ firewall: portgroups: *portgroupsdocker elastic_agent_endpoint: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker From 014aeffb2af91889bc182a8dd4cbf215ceef820f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:56:33 -0400 Subject: [PATCH 26/26] add analyst back --- salt/firewall/defaults.yaml | 17 +++++++++++++++-- salt/firewall/soc_firewall.yaml | 13 ++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index ff776d309..9b8325a34 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1,5 +1,6 @@ firewall: hostgroups: + analyst: [] anywhere: - 0.0.0.0/0 beats_endpoint: [] @@ -215,9 +216,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - workstation: + analyst: portgroups: - - yum + - nginx customhostgroup0: portgroups: [] customhostgroup1: @@ -441,6 +442,9 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + analyst: + portgroups: + - nginx beats_endpoint: portgroups: - beats_5044 @@ -626,6 +630,9 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + analyst: + portgroups: + - nginx beats_endpoint: portgroups: - beats_5044 @@ -816,6 +823,9 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + analyst: + portgroups: + - nginx beats_endpoint: portgroups: - beats_5044 @@ -1187,6 +1197,9 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + analyst: + portgroups: + - nginx workstation: portgroups: - yum diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 27c52e123..8f8dbb69d 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -1,6 +1,6 @@ firewall: hostgroups: - workstation: &hostgroupsettings + analyst: &hostgroupsettings description: List of IP or CIDR blocks to allow access to this hostgroup. forcedType: "[]string" helplink: firewall.html @@ -45,6 +45,7 @@ firewall: standalone: *hostgroupsettings strelka_frontend: *hostgroupsettings syslog: *hostgroupsettings + workstation: *hostgroupsettings customhostgroup0: &customhostgroupsettings description: List of IP or CIDR blocks to allow to this hostgroup. forcedType: "[]string" @@ -213,6 +214,8 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -361,6 +364,8 @@ firewall: portgroups: *portgroupsdocker endgame: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -450,6 +455,8 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -545,6 +552,8 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -811,6 +820,8 @@ firewall: portgroups: *portgroupsdocker elastic_agent_endpoint: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: