From cfc33b1a34acf93f1ee6d93c3bbda4c98ee0777c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 28 Jan 2024 10:12:25 -0500 Subject: [PATCH 01/63] Sync Elastic Agent Artifacts --- files/salt/master/master | 3 ++- salt/elasticfleet/enabled.sls | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/files/salt/master/master b/files/salt/master/master index b93fa93de..e9b36868c 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,7 +41,8 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt - + elasticartifacts: + - /nsm/elastic-fleet/artifacts # The master_roots setting configures a master-only copy of the file_roots dictionary, # used by the state compiler. diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index fef85d24c..bd8ab51c0 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -41,6 +41,15 @@ so-elastic-fleet-auto-configure-elasticsearch-urls: - retry: True {% endif %} +# Sync Elastic Agent artifacts to Fleet Node +{% if grains.role in ['so-fleet'] %} +elasticagent_syncartifacts: + file.recurse: + - name: /nsm/elastic-fleet/artifacts/beats + - source: salt://beats?saltenv=elasticartifacts + +{% endif %} + {% if SERVICETOKEN != '' %} so-elastic-fleet: docker_container.running: From 1847e5c3c0093751c6826df33c2b88a70933ea3c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 28 Jan 2024 11:37:18 -0500 Subject: [PATCH 02/63] Enable nginx on Fleet Node --- salt/allowed_states.map.jinja | 1 + salt/nginx/etc/nginx.conf | 20 ++++++++++++++++++++ salt/top.sls | 1 + 3 files changed, 22 insertions(+) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index d27f51ede..3ead8b26e 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -180,6 +180,7 @@ 'telegraf', 'firewall', 'logstash', + 'nginx', 'healthcheck', 'schedule', 'elasticfleet', diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index d5981be77..236f8da7f 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -39,6 +39,26 @@ http { include /etc/nginx/conf.d/*.conf; + {%- if role in ['fleet'] %} + + server { + listen 8443; + server_name {{ GLOBALS.hostname }}; + root /opt/socore/html; + location /artifacts/ { + try_files $uri =206; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + + {%- endif %} + {%- if role in ['eval', 'managersearch', 'manager', 'standalone', 'import'] %} server { diff --git a/salt/top.sls b/salt/top.sls index f8979956e..16b355476 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -264,6 +264,7 @@ base: - telegraf - firewall - logstash + - nginx - elasticfleet - elasticfleet.install_agent_grid - schedule From afa98fa147f9b66a97692a35917f6dc766b3b93f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 28 Jan 2024 14:20:52 -0500 Subject: [PATCH 03/63] update artifacts URL automatically --- salt/elasticfleet/enabled.sls | 7 +- .../so-elastic-fleet-artifacts-url-update | 102 ++++++++++++++++++ salt/firewall/containers.map.jinja | 1 + salt/nginx/enabled.sls | 5 + 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index bd8ab51c0..dca7f479f 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -33,12 +33,17 @@ so-elastic-fleet-auto-configure-server-urls: - retry: True {% endif %} -# Automatically update Fleet Server Elasticsearch URLs +# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs {% if grains.role not in ['so-fleet'] %} so-elastic-fleet-auto-configure-elasticsearch-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-es-url-update - retry: True + +so-elastic-fleet-auto-configure-elasticsearch-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-artifacts-url-update + - retry: True {% endif %} # Sync Elastic Agent artifacts to Fleet Node diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update new file mode 100644 index 000000000..685db392f --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -0,0 +1,102 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +########## +# Set Elastic Agent Artifact Registry URL + + + + +function update_es_urls() { + +# For each element in NEWLIST, create a new entry + +JSON_STRING=$( jq -n \ + --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ + --arg URL "http://{{ GLOBALS.url_base }}:8443/artifacts/" \ + '{"name":$NAME,"host":$URL,"is_default":true}' + ) + +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + +######### + + # Generate updated JSON payload +{% if grains.role not in ['so-import', 'so-eval'] %} + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}') +{%- else %} + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') +{%- endif %} + # Update Fleet Elasticsearch URLs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} + +#START HERE + +# Get current list of Artifact URLs +#RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch') +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources') + + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON") +if [ "$CHECKSUM" != "1" ]; then + printf "Failed to query for current Elastic Agent Artifact URLs..." + exit 1 +fi + +# Get the current list of Elastic Agent Artifact URLs & hash them +CURRENT_LIST=$(jq -c -r '.items[].host' <<< "$RAW_JSON") +CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + + +# Create array & add initial elements +if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then + NEW_LIST=("http://{{ GLOBALS.url_base }}:8443/artifacts/") +else + NEW_LIST=("http://{{ GLOBALS.url_base }}:8443/artifacts/" "http://{{ GLOBALS.hostname }}:8443/artifacts/") +fi + +# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) +LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') + +# Query for Fleet Nodes & add them to the list (Hostname) +if grep -q "fleet" <<< $LOGSTASHNODES; then + readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${FLEETNODES[@]}" + do + NEW_LIST+=("http://$NODE:8443/artifacts/") + done +fi + +# Sort & hash the new list of Fleet Elasticsearch URLs +NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") +NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') + +# Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs +if [ "$1" = "--force" ]; then + printf "\nUpdating List, since --force was specified.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_es_urls + exit 0 +fi + +if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then + printf "\nHashes match - no update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + exit 0 +else + printf "\nHashes don't match - update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + #update_es_urls +fi diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 0ba2389e9..99a3bd5d0 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -95,6 +95,7 @@ {% set NODE_CONTAINERS = [ 'so-elastic-fleet', 'so-logstash', + 'so-nginx' ] %} {% elif GLOBALS.role == 'so-sensor' %} diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index dda475655..eca9c237a 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -14,6 +14,9 @@ include: - nginx.config - nginx.sostatus + +{% if grains.role not in ['so-fleet'] %} + {# if the user has selected to replace the crt and key in the ui #} {% if NGINXMERGED.ssl.replace_cert %} @@ -88,6 +91,8 @@ make-rule-dir-nginx: - recurse: - user - group + +{% endif %} so-nginx: docker_container.running: From 0d08bb0a91f817efd7d66a8f818706c9b8afe65d Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 29 Jan 2024 11:37:28 -0500 Subject: [PATCH 04/63] Finalize script --- salt/elasticfleet/enabled.sls | 2 +- .../so-elastic-fleet-artifacts-url-update | 129 +++++++----------- 2 files changed, 51 insertions(+), 80 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index dca7f479f..f5f53c2a5 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -40,7 +40,7 @@ so-elastic-fleet-auto-configure-elasticsearch-urls: - name: /usr/sbin/so-elastic-fleet-es-url-update - retry: True -so-elastic-fleet-auto-configure-elasticsearch-urls: +so-elastic-fleet-auto-configure-artifact-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-artifacts-url-update - retry: True diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update index 685db392f..bcd3ef7f7 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -11,92 +11,63 @@ if ! is_manager_node; then exit 0 fi -########## -# Set Elastic Agent Artifact Registry URL - - - - -function update_es_urls() { - -# For each element in NEWLIST, create a new entry - -JSON_STRING=$( jq -n \ - --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ - --arg URL "http://{{ GLOBALS.url_base }}:8443/artifacts/" \ - '{"name":$NAME,"host":$URL,"is_default":true}' - ) - -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" - -######### - - # Generate updated JSON payload -{% if grains.role not in ['so-import', 'so-eval'] %} - JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}') -{%- else %} - JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') -{%- endif %} - # Update Fleet Elasticsearch URLs - curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +# Function to check if an array contains a value +array_contains () { + local array="$1[@]" + local seeking=$2 + local in=1 + for element in "${!array}"; do + if [[ $element == "$seeking" ]]; then + in=0 + break + fi + done + return $in } -#START HERE - -# Get current list of Artifact URLs -#RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch') -RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources') - - -# Check to make sure that the server responded with good data - else, bail from script -CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON") -if [ "$CHECKSUM" != "1" ]; then - printf "Failed to query for current Elastic Agent Artifact URLs..." - exit 1 -fi - -# Get the current list of Elastic Agent Artifact URLs & hash them -CURRENT_LIST=$(jq -c -r '.items[].host' <<< "$RAW_JSON") -CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') - - -# Create array & add initial elements -if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then - NEW_LIST=("http://{{ GLOBALS.url_base }}:8443/artifacts/") -else - NEW_LIST=("http://{{ GLOBALS.url_base }}:8443/artifacts/" "http://{{ GLOBALS.hostname }}:8443/artifacts/") -fi - # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') +# Initialize an array for new hosts from Fleet Nodes +declare -a NEW_LIST=() + # Query for Fleet Nodes & add them to the list (Hostname) -if grep -q "fleet" <<< $LOGSTASHNODES; then - readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) - for NODE in "${FLEETNODES[@]}" - do - NEW_LIST+=("http://$NODE:8443/artifacts/") - done +if grep -q "fleet" <<< "$LOGSTASHNODES"; then + readarray -t FLEETNODES < <(jq -r '.fleet | keys_unsorted[]' <<< "$LOGSTASHNODES") + for NODE in "${FLEETNODES[@]}"; do + NEW_LIST+=("http://$NODE:8443/artifacts/") + done fi -# Sort & hash the new list of Fleet Elasticsearch URLs -NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") -NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') +# Create an array for expected hosts and their names +declare -A expected_hosts=( + ["http://{{ GLOBALS.url_base }}:8443/artifacts/"]="FleetServer_{{ GLOBALS.hostname }}" + ["https://artifacts.elastic.co/downloads/"]="Elastic Artifacts" +) -# Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs -if [ "$1" = "--force" ]; then - printf "\nUpdating List, since --force was specified.\n" - printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" - update_es_urls - exit 0 -fi +# Merge NEW_LIST into expected_hosts +for host in "${NEW_LIST[@]}"; do + expected_hosts[$host]="FleetServer" +done -if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then - printf "\nHashes match - no update needed.\n" - printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" - exit 0 -else - printf "\nHashes don't match - update needed.\n" - printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" - #update_es_urls -fi +# Fetch the current hosts from the API +current_hosts=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host) + +# Convert current hosts to an array +IFS=$'\n' read -rd '' -a current_hosts_array <<<"$current_hosts" + +# Check each expected host +for host in "${!expected_hosts[@]}"; do + array_contains current_hosts_array "$host" || { + echo "$host (${expected_hosts[$host]}) is missing. Adding it..." + + # Prepare the JSON payload + JSON_STRING=$( jq -n \ + --arg NAME "${expected_hosts[$host]}" \ + --arg URL "$host" \ + '{"name":$NAME,"host":$URL}' ) + + # Create the missing host + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + } +done From 1192dbd5305c190460effdc28ccbd85d79427cf7 Mon Sep 17 00:00:00 2001 From: Pete Date: Wed, 31 Jan 2024 09:01:56 -0500 Subject: [PATCH 05/63] also remove intca symlink The symlink is created in init.sls; it should be removed here. --- salt/ssl/remove.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/ssl/remove.sls b/salt/ssl/remove.sls index 43a245288..7738490e5 100644 --- a/salt/ssl/remove.sls +++ b/salt/ssl/remove.sls @@ -2,6 +2,10 @@ trusttheca: file.absent: - name: /etc/pki/tls/certs/intca.crt +symlinkca: + file.absent: + - name: /etc/ssl/certs/intca.crt + influxdb_key: file.absent: - name: /etc/pki/influxdb.key From cd4bd6460aa117ebc798c6f74f5ddfa32b6fc65f Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:16:18 +0000 Subject: [PATCH 06/63] Custom pipelines --- salt/elasticsearch/defaults.yaml | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ce1bfb08d..541f3fae7 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -55,6 +55,87 @@ elasticsearch: key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none enabled: false + pipelines: + custom01: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom01 + - pipeline: + name: common + custom02: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom02 + - pipeline: + name: common + custom03: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom03 + - pipeline: + name: common + custom04: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom04 + - pipeline: + name: common + custom05: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom05 + - pipeline: + name: common + custom06: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom06 + - pipeline: + name: common + custom07: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom07 + - pipeline: + name: common + custom08: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom08 + - pipeline: + name: common + custom09: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom09 + - pipeline: + name: common + custom10: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom10 + - pipeline: + name: common index_settings: global_overrides: index_template: From bc75be940243793ee0b33c031efbcc00c2c988e3 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:16:48 +0000 Subject: [PATCH 07/63] Custom pipelines in UI --- salt/elasticsearch/config.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 23e11a710..27a8a0fd6 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,19 @@ esingestconf: - user: 930 - group: 939 +# Auto-generate Elasticsearch ingest node pipelines from pillar +{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} +es_ingest_conf_{{pipeline}}: + file.managed: + - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }} + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ config }} + - template: jinja + - onchanges_in: + - file: so-pipelines-reload +{% endfor %} + eslog4jfile: file.managed: - name: /opt/so/conf/elasticsearch/log4j2.properties From 1853dc398bc377dc367cd0aaeeaefece7acec3c5 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:17:33 +0000 Subject: [PATCH 08/63] Custom pipeline configuration --- salt/logstash/defaults.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index e4c18cc64..2cafce6fd 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -42,6 +42,24 @@ logstash: custom2: [] custom3: [] custom4: [] + pipeline_config: + custom01: |- + filter { + if [event][module] =~ "zeek" { + mutate { + add_tag => ["network_stuff"] + } + } + } + custom02: PLACEHOLDER + custom03: PLACEHOLDER + custom04: PLACEHOLDER + custom05: PLACEHOLDER + custom06: PLACEHOLDER + custom07: PLACEHOLDER + custom08: PLACEHOLDER + custom09: PLACEHOLDER + custom10: PLACEHOLDER settings: lsheap: 500m config: From 4672a5b8ebe83e4c5abac79abf5e87ab3c1f3d39 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:18:17 +0000 Subject: [PATCH 09/63] Custom pipeline configuration in UI --- salt/logstash/config.sls | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 6ff33ff1a..8a59c83b7 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -63,6 +63,20 @@ lspipelinedir: - user: 931 - group: 939 +# Auto-generate Logstash pipeline config +{% for pipeline, config in LOGSTASH_MERGED.pipeline_config.items() %} +{% for assigned_pipeline in ASSIGNED_PIPELINES %} +{% set custom_pipeline = 'custom/' + pipeline + '.conf' %} +{% if custom_pipeline in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: + file.managed: + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{ pipeline }}.conf + - contents: LOGSTASH_MERGED.pipeline_config.{{pipeline}} +{% endif %} +{% endfor %} +{% endfor %} + + {% for assigned_pipeline in ASSIGNED_PIPELINES %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: From 0fe96bfc2d66223f2d3ffa1ad4ee30ed5b1144fc Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 31 Jan 2024 16:17:40 -0500 Subject: [PATCH 10/63] switch to symlink --- salt/elasticfleet/enabled.sls | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index f5f53c2a5..a84d51cfc 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -44,6 +44,13 @@ so-elastic-fleet-auto-configure-artifact-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-artifacts-url-update - retry: True + +elasticagentartifactssymlink: + file.symlink: + - name: /opt/so/saltstack/local/salt/beats + - target: /nsm/elastic-fleet/artifacts/beats + - user: socore + - group: socore {% endif %} # Sync Elastic Agent artifacts to Fleet Node @@ -51,8 +58,7 @@ so-elastic-fleet-auto-configure-artifact-urls: elasticagent_syncartifacts: file.recurse: - name: /nsm/elastic-fleet/artifacts/beats - - source: salt://beats?saltenv=elasticartifacts - + - source: salt://beats {% endif %} {% if SERVICETOKEN != '' %} From 2f03248612bc7e881733d1eec9201d8e70f3fd7c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 31 Jan 2024 16:22:44 -0500 Subject: [PATCH 11/63] use different nginx defaults for so-fleet node hosting artifacts --- salt/docker/defaults.yaml | 7 +++++++ salt/firewall/containers.map.jinja | 2 +- salt/nginx/enabled.sls | 23 +++++++++++++++-------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 9a27843ae..4bc212fbe 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -84,6 +84,13 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] + 'so-nginx-fleet-node': + final_octet: 31 + port_bindings: + - 8443:8443 + custom_bind_mounts: [] + extra_hosts: [] + extra_env: [] 'so-playbook': final_octet: 32 port_bindings: diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 99a3bd5d0..b3ead0f4c 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -95,7 +95,7 @@ {% set NODE_CONTAINERS = [ 'so-elastic-fleet', 'so-logstash', - 'so-nginx' + 'so-nginx-fleet-node' ] %} {% elif GLOBALS.role == 'so-sensor' %} diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index eca9c237a..273fb65be 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -94,17 +94,24 @@ make-rule-dir-nginx: {% endif %} +{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #} +{% if GLOBALS.role == 'so-fleet' %} +{% set container_config = 'so-nginx-fleet-node' %} +{% else %} +{% set container_config = 'so-nginx' %} +{% endif %} + so-nginx: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }} - hostname: so-nginx - networks: - sobridge: - - ipv4_address: {{ DOCKER.containers['so-nginx'].ip }} + - ipv4_address: {{ DOCKER.containers[container_config].ip }} - extra_hosts: - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} - {% if DOCKER.containers['so-nginx'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-nginx'].extra_hosts %} + {% if DOCKER.containers[container_config].extra_hosts %} + {% for XTRAHOST in DOCKER.containers[container_config].extra_hosts %} - {{ XTRAHOST }} {% endfor %} {% endif %} @@ -124,20 +131,20 @@ so-nginx: - /nsm/repo:/opt/socore/html/repo:ro - /nsm/rules:/nsm/rules:ro {% endif %} - {% if DOCKER.containers['so-nginx'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-nginx'].custom_bind_mounts %} + {% if DOCKER.containers[container_config].custom_bind_mounts %} + {% for BIND in DOCKER.containers[container_config].custom_bind_mounts %} - {{ BIND }} {% endfor %} {% endif %} - {% if DOCKER.containers['so-nginx'].extra_env %} + {% if DOCKER.containers[container_config].extra_env %} - environment: - {% for XTRAENV in DOCKER.containers['so-nginx'].extra_env %} + {% for XTRAENV in DOCKER.containers[container_config].extra_env %} - {{ XTRAENV }} {% endfor %} {% endif %} - cap_add: NET_BIND_SERVICE - port_bindings: - {% for BINDING in DOCKER.containers['so-nginx'].port_bindings %} + {% for BINDING in DOCKER.containers[container_config].port_bindings %} - {{ BINDING }} {% endfor %} - watch: From ae32ac40c2dde62a0c26319c78a667700406f94e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 31 Jan 2024 16:28:45 -0500 Subject: [PATCH 12/63] add fleet node nginx to docker annotations --- salt/docker/soc_docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index 850324a9e..6e0efeb20 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -48,6 +48,7 @@ docker: so-logstash: *dockerOptions so-mysql: *dockerOptions so-nginx: *dockerOptions + so-nginx-fleet-node: *dockerOptions so-playbook: *dockerOptions so-redis: *dockerOptions so-sensoroni: *dockerOptions From bc502cc065c934d1afad9c08c4001a5bcb9dd732 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 21:46:33 +0000 Subject: [PATCH 13/63] Custom Elasticserach pipeline annotations --- salt/elasticsearch/soc_elasticsearch.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index c54e07660..0f410e716 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -45,6 +45,28 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html + pipelines: + custom01: &pipelines + description: + description: Description of the ingest node pipeline + global: True + advanced: True + helpLink: elasticsearch.html + processors: + description: Processors for the ingest node pipeline + global: True + advanced: True + multiline: True + helpLink: elasticsearch.html + custom02: *pipelines + custom03: *pipelines + custom04: *pipelines + custom05: *pipelines + custom06: *pipelines + custom07: *pipelines + custom08: *pipelines + custom09: *pipelines + custom10: *pipelines index_settings: global_overrides: index_template: From 136097f9816b30b85804a2b43ddf208205c28581 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 21:47:09 +0000 Subject: [PATCH 14/63] Custom Logstash pipeline annotations --- salt/logstash/soc_logstash.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index bcb99bad5..dc52d63a8 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -31,6 +31,22 @@ logstash: custom2: *defined_pipelines custom3: *defined_pipelines custom4: *defined_pipelines + pipeline_config: + custom01: &pipeline_config + description: Pipeline configuration for Logstash + advanced: True + multiline: True + forcedType: string + helpLink: logstash.html + custom02: *pipeline_config + custom03: *pipeline_config + custom04: *pipeline_config + custom05: *pipeline_config + custom06: *pipeline_config + custom07: *pipeline_config + custom08: *pipeline_config + custom09: *pipeline_config + custom10: *pipeline_config settings: lsheap: description: Heap size to use for logstash From 182667bafb3769b636f715385d24805dda40c2bb Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 1 Feb 2024 13:59:23 +0000 Subject: [PATCH 15/63] Change numbers for Elasticsearch --- salt/elasticsearch/defaults.yaml | 40 +++++++++++------------ salt/elasticsearch/soc_elasticsearch.yaml | 20 ++++++------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 541f3fae7..03cd6d519 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,84 +56,84 @@ elasticsearch: verification_mode: none enabled: false pipelines: - custom01: + custom001: description: Custom Pipeline processors: - set: field: tags - value: custom01 + value: custom001 - pipeline: name: common - custom02: + custom002: description: Custom Pipeline processors: - set: field: tags - value: custom02 + value: custom002 - pipeline: name: common - custom03: + custom003: description: Custom Pipeline processors: - set: field: tags - value: custom03 + value: custom003 - pipeline: name: common - custom04: + custom004: description: Custom Pipeline processors: - set: field: tags - value: custom04 + value: custom004 - pipeline: name: common - custom05: + custom005: description: Custom Pipeline processors: - set: field: tags - value: custom05 + value: custom005 - pipeline: name: common - custom06: + custom006: description: Custom Pipeline processors: - set: field: tags - value: custom06 + value: custom006 - pipeline: name: common - custom07: + custom007: description: Custom Pipeline processors: - set: field: tags - value: custom07 + value: custom007 - pipeline: name: common - custom08: + custom008: description: Custom Pipeline processors: - set: field: tags - value: custom08 + value: custom008 - pipeline: name: common - custom09: + custom009: description: Custom Pipeline processors: - set: field: tags - value: custom09 + value: custom009 - pipeline: name: common - custom10: + custom010: description: Custom Pipeline processors: - set: field: tags - value: custom10 + value: custom010 - pipeline: name: common index_settings: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 0f410e716..9a64190b3 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -46,7 +46,7 @@ elasticsearch: global: True helpLink: elasticsearch.html pipelines: - custom01: &pipelines + custom001: &pipelines description: description: Description of the ingest node pipeline global: True @@ -58,15 +58,15 @@ elasticsearch: advanced: True multiline: True helpLink: elasticsearch.html - custom02: *pipelines - custom03: *pipelines - custom04: *pipelines - custom05: *pipelines - custom06: *pipelines - custom07: *pipelines - custom08: *pipelines - custom09: *pipelines - custom10: *pipelines + custom002: *pipelines + custom003: *pipelines + custom004: *pipelines + custom005: *pipelines + custom006: *pipelines + custom007: *pipelines + custom008: *pipelines + custom009: *pipelines + custom010: *pipelines index_settings: global_overrides: index_template: From 1818e134cafe3c29224d66cd623f839d2065e8cd Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 1 Feb 2024 14:01:55 +0000 Subject: [PATCH 16/63] Change numbers for Logstash --- salt/logstash/defaults.yaml | 20 ++++++++++---------- salt/logstash/soc_logstash.yaml | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 2cafce6fd..348acb622 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -43,7 +43,7 @@ logstash: custom3: [] custom4: [] pipeline_config: - custom01: |- + custom001: |- filter { if [event][module] =~ "zeek" { mutate { @@ -51,15 +51,15 @@ logstash: } } } - custom02: PLACEHOLDER - custom03: PLACEHOLDER - custom04: PLACEHOLDER - custom05: PLACEHOLDER - custom06: PLACEHOLDER - custom07: PLACEHOLDER - custom08: PLACEHOLDER - custom09: PLACEHOLDER - custom10: PLACEHOLDER + custom002: PLACEHOLDER + custom003: PLACEHOLDER + custom004: PLACEHOLDER + custom005: PLACEHOLDER + custom006: PLACEHOLDER + custom007: PLACEHOLDER + custom008: PLACEHOLDER + custom009: PLACEHOLDER + custom010: PLACEHOLDER settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index dc52d63a8..3172ff7c5 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -32,21 +32,21 @@ logstash: custom3: *defined_pipelines custom4: *defined_pipelines pipeline_config: - custom01: &pipeline_config + custom001: &pipeline_config description: Pipeline configuration for Logstash advanced: True multiline: True forcedType: string helpLink: logstash.html - custom02: *pipeline_config - custom03: *pipeline_config - custom04: *pipeline_config - custom05: *pipeline_config - custom06: *pipeline_config - custom07: *pipeline_config - custom08: *pipeline_config - custom09: *pipeline_config - custom10: *pipeline_config + custom002: *pipeline_config + custom003: *pipeline_config + custom004: *pipeline_config + custom005: *pipeline_config + custom006: *pipeline_config + custom007: *pipeline_config + custom008: *pipeline_config + custom009: *pipeline_config + custom010: *pipeline_config settings: lsheap: description: Heap size to use for logstash From e090518b5920ef593e9bbb336a938edb6afe3a0e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 Feb 2024 09:46:53 -0500 Subject: [PATCH 17/63] Refactor script --- .../so-elastic-fleet-artifacts-url-update | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update index bcd3ef7f7..ffbeaf6e1 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -26,7 +26,7 @@ array_contains () { } # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) -LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') +LOGSTASHNODES='{{ salt['pillar.get']('logstash:nodes', {}) | tojson }}' # Initialize an array for new hosts from Fleet Nodes declare -a NEW_LIST=() @@ -40,34 +40,46 @@ if grep -q "fleet" <<< "$LOGSTASHNODES"; then fi # Create an array for expected hosts and their names -declare -A expected_hosts=( +declare -A expected_urls=( ["http://{{ GLOBALS.url_base }}:8443/artifacts/"]="FleetServer_{{ GLOBALS.hostname }}" ["https://artifacts.elastic.co/downloads/"]="Elastic Artifacts" ) -# Merge NEW_LIST into expected_hosts +# Merge NEW_LIST into expected_urls for host in "${NEW_LIST[@]}"; do - expected_hosts[$host]="FleetServer" + expected_urls[$host]="FleetServer" done # Fetch the current hosts from the API -current_hosts=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host) +current_urls=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host) # Convert current hosts to an array -IFS=$'\n' read -rd '' -a current_hosts_array <<<"$current_hosts" +IFS=$'\n' read -rd '' -a current_urls_array <<<"$current_urls" + +# Flag to track if any host was added +any_url_added=0 # Check each expected host -for host in "${!expected_hosts[@]}"; do - array_contains current_hosts_array "$host" || { - echo "$host (${expected_hosts[$host]}) is missing. Adding it..." +for host in "${!expected_urls[@]}"; do + array_contains current_urls_array "$host" || { + echo "$host (${expected_urls[$host]}) is missing. Adding it..." # Prepare the JSON payload JSON_STRING=$( jq -n \ - --arg NAME "${expected_hosts[$host]}" \ + --arg NAME "${expected_urls[$host]}" \ --arg URL "$host" \ '{"name":$NAME,"host":$URL}' ) # Create the missing host curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + + # Flag that an artifact URL was added + any_url_added=1 } + done + + +if [[ $any_url_added -eq 0 ]]; then + echo "All expected artifact URLs are present. No updates needed." +fi From 0d5db58c86f39521ec954d186db6050b7d41fbad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Feb 2024 10:32:41 -0500 Subject: [PATCH 18/63] upgrade salt3006.6 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 1b4d2e63a..19677f70b 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.5 + version: 3006.6 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index c15929951..2e4ebc93e 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.5 + version: 3006.6 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From 7a29b3a529cd3ddb60e2b2a144d6b8861cac524e Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 2 Feb 2024 08:45:01 -0500 Subject: [PATCH 19/63] call salt before stopping salt services salt-call does not work when the salt-master is not running. If these calls are to succeed, they should occur before the salt services are stopped. --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d19f27620..717ec0ce4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1600,6 +1600,9 @@ reinstall_init() { salt-call -l info saltutil.kill_all_jobs --local fi + logCmd "salt-call state.apply ca.remove -linfo --local --file-root=../salt" + logCmd "salt-call state.apply ssl.remove -linfo --local --file-root=../salt" + # Kill any salt processes (safely) for service in "${salt_services[@]}"; do # Stop the service in the background so we can exit after a certain amount of time @@ -1621,9 +1624,6 @@ reinstall_init() { done done - logCmd "salt-call state.apply ca.remove -linfo --local --file-root=../salt" - logCmd "salt-call state.apply ssl.remove -linfo --local --file-root=../salt" - # Remove all salt configs rm -rf /etc/salt/engines/* /etc/salt/grains /etc/salt/master /etc/salt/master.d/* /etc/salt/minion /etc/salt/minion.d/* /etc/salt/pki/* /etc/salt/proxy /etc/salt/proxy.d/* /var/cache/salt/ From cf83d1cb869a5c2a53d3a948e9193c152261fec6 Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 2 Feb 2024 12:25:16 -0500 Subject: [PATCH 20/63] feat: use mountpoint for Elastic log limit Instead of just existence, this checks if the directories are separate mountpoints when determining disk size and log_size_limit calculations. It also sets the percentage to 80 if /nsm/elasticsearch is a separate mountpoint. This allows for better disk utilization on server configurations where /nsm is based on large slow HDDs for increased PCAP retention but /nsm/elasticsearch is based on SSDs for faster Elasticsearch performance. --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d19f27620..3caf76c06 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2148,11 +2148,12 @@ set_default_log_size() { esac local disk_dir="/" - if [ -d /nsm ]; then + if mountpoint -q /nsm; then disk_dir="/nsm" fi - if [ -d /nsm/elasticsearch ]; then + if mountpoint -q /nsm/elasticsearch; then disk_dir="/nsm/elasticsearch" + percentage=80 fi local disk_size_1k From 2643ae08a75c097abc1de60397f33b72a66655f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Feb 2024 17:54:30 -0500 Subject: [PATCH 21/63] add append to list --- salt/manager/tools/sbin/so-yaml.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 874fc9e0f..572585df9 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -16,12 +16,14 @@ lockFile = "/tmp/so-yaml.lock" def showUsage(args): print('Usage: {} [ARGS...]'.format(sys.argv[0])) print(' General commands:') + print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.') print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') print(' help - Prints this usage information.') print('') print(' Where:') print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') + print(' LISTITEM - Item to add to the list.') sys.exit(1) @@ -35,6 +37,32 @@ def writeYaml(filename, content): file = open(filename, "w") return yaml.dump(content, file) +def appendItem(content, key, listItem): + pieces = key.split(".", 1) + if len(pieces) > 1: + appendItem(content[pieces[0]], pieces[1], listItem) + else: + try: + content[key].append(listItem) + except TypeError: + print("The contents key provided is likely not a list. No action was taken on the file.") + return 1 + +def append(args): + if len(args) != 3: + print('Missing filename, key arg, or list item to append', file=sys.stderr) + showUsage(None) + return + + filename = args[0] + key = args[1] + listItem = args[2] + + content = loadYaml(filename) + appendItem(content, key, listItem) + writeYaml(filename, content) + + return 0 def removeKey(content, key): pieces = key.split(".", 1) @@ -69,6 +97,7 @@ def main(): commands = { "help": showUsage, + "append": append, "remove": remove, } From 9d62ade32e72f10e058428c1e441b0de225d8527 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 11:14:27 -0500 Subject: [PATCH 22/63] update so-yaml tests --- salt/manager/tools/sbin/so-yaml.py | 2 +- salt/manager/tools/sbin/so-yaml_test.py | 44 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 572585df9..9f4bdbb98 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -45,7 +45,7 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except TypeError: - print("The contents key provided is likely not a list. No action was taken on the file.") + print("The key provided is likely not a list. No action was taken on the file.") return 1 def append(args): diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7d0ed1a8e..f70314ba3 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -105,3 +105,47 @@ class TestRemove(unittest.TestCase): self.assertEqual(actual, expected) sysmock.assert_called_once_with(1) self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + + def test_append(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}") + file.close() + + soyaml.append([filename, "key3", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + expected = "key1:\n child1: 123\n child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\n- d\n" + self.assertEqual(actual, expected) + + def test_append_nested(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_append_nested_deep(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2.deep2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) From 7106095128e643a4e50891c4631a406c35479cc8 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 6 Feb 2024 15:39:23 -0500 Subject: [PATCH 23/63] FEATURE: Improve Correlate and Hunt actions on SOC Actions menu #12315 --- salt/soc/defaults.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c1b9470c8..86bb57c49 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -9,7 +9,7 @@ soc: icon: fa-crosshairs target: links: - - '/#/hunt?q="{value|escape}" | groupby event.module* event.dataset' + - '/#/hunt?q="{value|escape}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - name: actionAddToCase description: actionAddToCaseHelp icon: fa-briefcase @@ -23,13 +23,13 @@ soc: icon: fab fa-searchengin target: '' links: - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* event.dataset' - - '/#/hunt?q="{:log.id.uid}" | groupby event.module* event.dataset' - - '/#/hunt?q="{:network.community_id}" | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:log.id.uid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:network.community_id}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - name: actionPcap description: actionPcapHelp icon: fa-stream From b3f61536670050996fd539444863aa95a203a2b9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 16:15:54 -0500 Subject: [PATCH 24/63] update so-yaml tests --- salt/manager/tools/sbin/so-yaml.py | 7 +++- salt/manager/tools/sbin/so-yaml_test.py | 52 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 9f4bdbb98..24d520891 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -44,8 +44,11 @@ def appendItem(content, key, listItem): else: try: content[key].append(listItem) - except TypeError: - print("The key provided is likely not a list. No action was taken on the file.") + except AttributeError: + print("The key provided is not a list. No action was taken on the file.") + return 1 + except KeyError: + print("The key provided does not exist. No action was taken on the file.") return 1 def append(args): diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index f70314ba3..59db43860 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -149,3 +149,55 @@ class TestRemove(unittest.TestCase): expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" self.assertEqual(actual, expected) + + def test_append_key_noexist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key4", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_noexist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_nonlist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + + def test_append_key_nonlist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") From 24fd3ef8cc40a4b0f3e7083ae935171079de11f8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 16:22:13 -0500 Subject: [PATCH 25/63] uopdate error message --- salt/manager/tools/sbin/so-yaml.py | 2 +- salt/manager/tools/sbin/so-yaml_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 24d520891..41cab0b23 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -45,7 +45,7 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except AttributeError: - print("The key provided is not a list. No action was taken on the file.") + print("The existing value for the given key is not a list. No action was taken on the file.") return 1 except KeyError: print("The key provided does not exist. No action was taken on the file.") diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 59db43860..488877ea1 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -187,7 +187,7 @@ class TestRemove(unittest.TestCase): sys.argv = ["cmd", "append", filename, "key1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") def test_append_key_nonlist_deep(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -200,4 +200,4 @@ class TestRemove(unittest.TestCase): sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") From f97d0f2f3644b298ef14c313fd9e94c39baa9573 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 09:25:56 -0500 Subject: [PATCH 26/63] add /opt/so/rules/ to files_roots --- files/salt/master/master | 1 + salt/manager/tools/sbin/soup | 10 ++++++++++ salt/suricata/config.sls | 4 +++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/files/salt/master/master b/files/salt/master/master index b93fa93de..e309a560b 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,6 +41,7 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt + - /opt/so/rules # The master_roots setting configures a master-only copy of the file_roots dictionary, diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a250116d1..6f086469a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -594,6 +594,16 @@ up_to_2.4.50() { touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls + # the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots + # put new so-yaml in place + echo "Updating so-yaml" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ + echo "Creating a backup of the salt-master config." + cp -v /etc/salt/master "/etc/salt/master.so-$INSTALLEDVERSION" + echo "Adding /opt/so/rules to file_roots using so-yaml" + so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules + INSTALLEDVERSION=2.4.50 } diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 8d5279349..4804565ce 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -84,10 +84,12 @@ suridatadir: - mode: 770 - makedirs: True +# salt:// would resolve to /opt/so/rules because of the defined file_roots and +# nids not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt surirulesync: file.recurse: - name: /opt/so/conf/suricata/rules/ - - source: salt://suricata/rules/ + - source: salt://nids/ - user: 940 - group: 940 - show_changes: False From e42e07b245d32dd2bdb1d54e807d478209506efa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 13:05:45 -0500 Subject: [PATCH 27/63] update salt mine after salt-master restarts --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 6f086469a..b4647bdbc 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -947,9 +947,6 @@ main() { systemctl_func "stop" "$cron_service_name" - # update mine items prior to stopping salt-minion and salt-master - update_salt_mine - echo "Updating dockers to $NEWVERSION." if [[ $is_airgap -eq 0 ]]; then airgap_update_dockers @@ -1025,6 +1022,9 @@ main() { salt-call state.apply salt.minion -l info queue=True echo "" + # ensure the mine is updated and populated before highstates run, following the salt-master restart + update_salt_mine + enable_highstate echo "" From 6534f392a9334a2dd1f1b89b9961d116d5c44de7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 14:25:28 -0500 Subject: [PATCH 28/63] update backup filename --- salt/manager/tools/sbin/soup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b4647bdbc..5bade9891 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -600,8 +600,11 @@ up_to_2.4.50() { \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ echo "Creating a backup of the salt-master config." - cp -v /etc/salt/master "/etc/salt/master.so-$INSTALLEDVERSION" - echo "Adding /opt/so/rules to file_roots using so-yaml" + # INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version + # so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until + # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup + cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak" + echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules INSTALLEDVERSION=2.4.50 From d3d2305f00d90a2e076a8d377fcd9e72ea97a229 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 7 Feb 2024 16:08:27 -0500 Subject: [PATCH 29/63] FEATURE: Add new dashboards for community_id and firewall auth #12323 --- salt/soc/defaults.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 86bb57c49..31b6eb588 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1424,8 +1424,11 @@ soc: - name: Zeek Notice description: Zeek notice logs query: 'event.dataset:zeek.notice | groupby -sankey notice.note destination.ip | groupby notice.note | groupby notice.message | groupby notice.sub_message | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - - name: Connections - description: Network connection metadata + - name: Connections and Metadata with community_id + description: Network connections that include community_id + query: '_exists_:network.community_id | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - name: Connections seen by Zeek or Suricata + description: Network connections logged by Zeek or Suricata query: 'tags:conn | groupby source.ip | groupby destination.ip | groupby destination.port | groupby -sankey destination.port network.protocol | groupby network.protocol | groupby network.transport | groupby connection.history | groupby connection.state | groupby connection.state_description | groupby source.geo.country_name | groupby destination.geo.country_name | groupby client.ip_bytes | groupby server.ip_bytes | groupby client.oui' - name: DCE_RPC description: DCE_RPC (Distributed Computing Environment / Remote Procedure Calls) network metadata @@ -1562,6 +1565,9 @@ soc: - name: Firewall description: Firewall logs query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Firewall Auth + description: Firewall authentication logs + query: 'observer.type:firewall AND event.category:authentication | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | table soc_timestamp user.name source.ip message' - name: VLAN description: VLAN (Virtual Local Area Network) tagged logs query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name' From 8d0e8789bd949932df12397e3edafd4bfb4dd9e2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 8 Feb 2024 09:54:51 -0500 Subject: [PATCH 30/63] Use salt file roots --- salt/elasticfleet/enabled.sls | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index a84d51cfc..53ec6c1ef 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -45,12 +45,6 @@ so-elastic-fleet-auto-configure-artifact-urls: - name: /usr/sbin/so-elastic-fleet-artifacts-url-update - retry: True -elasticagentartifactssymlink: - file.symlink: - - name: /opt/so/saltstack/local/salt/beats - - target: /nsm/elastic-fleet/artifacts/beats - - user: socore - - group: socore {% endif %} # Sync Elastic Agent artifacts to Fleet Node @@ -58,7 +52,7 @@ elasticagentartifactssymlink: elasticagent_syncartifacts: file.recurse: - name: /nsm/elastic-fleet/artifacts/beats - - source: salt://beats + - source: salt://beats?saltenv=elasticartifacts {% endif %} {% if SERVICETOKEN != '' %} From 683abf0179701d7abfedbfd6d6d08d9227f8b949 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 8 Feb 2024 13:24:25 -0500 Subject: [PATCH 31/63] Rework naming --- .../sbin_jinja/so-elastic-fleet-artifacts-url-update | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update index ffbeaf6e1..721525668 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -35,7 +35,9 @@ declare -a NEW_LIST=() if grep -q "fleet" <<< "$LOGSTASHNODES"; then readarray -t FLEETNODES < <(jq -r '.fleet | keys_unsorted[]' <<< "$LOGSTASHNODES") for NODE in "${FLEETNODES[@]}"; do - NEW_LIST+=("http://$NODE:8443/artifacts/") + URL="http://$NODE:8443/artifacts/" + NAME="FleetServer_$NODE" + NEW_LIST+=("$URL=$NAME") done fi @@ -46,8 +48,11 @@ declare -A expected_urls=( ) # Merge NEW_LIST into expected_urls -for host in "${NEW_LIST[@]}"; do - expected_urls[$host]="FleetServer" +for entry in "${NEW_LIST[@]}"; do + # Extract URL and Name from each entry + IFS='=' read -r URL NAME <<< "$entry" + # Add to expected_urls, automatically handling URL as key and NAME as value + expected_urls["$URL"]="$NAME" done # Fetch the current hosts from the API From 3c9d6da1d8a81472aff15b56b1d2f75ce539acb9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 8 Feb 2024 22:05:37 -0500 Subject: [PATCH 32/63] add putty to sod packages.sls Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/desktop/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 9a7d53317..b2a028e60 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -334,6 +334,7 @@ desktop_packages: - pulseaudio-libs - pulseaudio-libs-glib2 - pulseaudio-utils + - putty - sane-airscan - sane-backends - sane-backends-drivers-cameras From 654602bf80fc344eadd21a861073066e260412e6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 9 Feb 2024 09:30:18 -0500 Subject: [PATCH 33/63] Fixup shell --- .../elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade | 2 ++ .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 2 ++ .../tools/sbin_jinja/so-elastic-fleet-outputs-update | 2 ++ salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update | 2 ++ 4 files changed, 8 insertions(+) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index b1ca8c476..b911f5896 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 5d5b7e7e0..3da6b3e78 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 23a206921..eb5ccc1ed 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 31c7becca..5f7637cd3 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. From 5903ae596cd3b322b79b86104c3fe0bb26d69cf3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 9 Feb 2024 09:47:23 -0500 Subject: [PATCH 34/63] move suricata rules to /opt/so/rules/nids/suri --- files/salt/master/master | 2 +- salt/idstools/enabled.sls | 2 +- salt/idstools/etc/rulecat.conf | 8 ++++---- salt/idstools/sync_files.sls | 4 ++-- salt/manager/tools/sbin/soup | 4 +++- salt/suricata/config.sls | 6 +++--- salt/suricata/manager.sls | 4 ++-- 7 files changed, 16 insertions(+), 14 deletions(-) diff --git a/files/salt/master/master b/files/salt/master/master index e309a560b..6cf3b94ea 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,7 +41,7 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt - - /opt/so/rules + - /opt/so/rules/nids # The master_roots setting configures a master-only copy of the file_roots dictionary, diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index decc5a5b2..5e4c4c066 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -39,7 +39,7 @@ so-idstools: {% endif %} - binds: - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - - /opt/so/rules/nids:/opt/so/rules/nids:rw + - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw - /nsm/rules/:/nsm/rules/:rw {% if DOCKER.containers['so-idstools'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %} diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf index d6f3d93d8..f7c784413 100644 --- a/salt/idstools/etc/rulecat.conf +++ b/salt/idstools/etc/rulecat.conf @@ -1,10 +1,10 @@ {%- from 'vars/globals.map.jinja' import GLOBALS -%} {%- from 'idstools/map.jinja' import IDSTOOLSMERGED -%} ---merged=/opt/so/rules/nids/all.rules ---local=/opt/so/rules/nids/local.rules +--merged=/opt/so/rules/nids/suri/all.rules +--local=/opt/so/rules/nids/suri/local.rules {%- if GLOBALS.md_engine == "SURICATA" %} ---local=/opt/so/rules/nids/extraction.rules ---local=/opt/so/rules/nids/filters.rules +--local=/opt/so/rules/nids/suri/extraction.rules +--local=/opt/so/rules/nids/suri/filters.rules {%- endif %} --url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules --disable=/opt/so/idstools/etc/disable.conf diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index 64479e937..cdacfaa74 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -21,7 +21,7 @@ idstoolsetcsync: rulesdir: file.directory: - - name: /opt/so/rules/nids + - name: /opt/so/rules/nids/suri - user: 939 - group: 939 - makedirs: True @@ -29,7 +29,7 @@ rulesdir: # Don't show changes because all.rules can be large synclocalnidsrules: file.recurse: - - name: /opt/so/rules/nids/ + - name: /opt/so/rules/nids/suri/ - source: salt://idstools/rules/ - user: 939 - group: 939 diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5bade9891..02c7c01e0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -605,7 +605,9 @@ up_to_2.4.50() { # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak" echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" - so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules + so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules/nids + echo "Moving Suricata rules" + mv -v /opt/so/rules/nids/* /opt/so/rules/nids/suri/. INSTALLEDVERSION=2.4.50 } diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 4804565ce..e0a157a85 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -84,12 +84,12 @@ suridatadir: - mode: 770 - makedirs: True -# salt:// would resolve to /opt/so/rules because of the defined file_roots and -# nids not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt +# salt:// would resolve to /opt/so/rules/nids because of the defined file_roots and +# not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt surirulesync: file.recurse: - name: /opt/so/conf/suricata/rules/ - - source: salt://nids/ + - source: salt://suricata/ - user: 940 - group: 940 - show_changes: False diff --git a/salt/suricata/manager.sls b/salt/suricata/manager.sls index c196c5cae..3d5183556 100644 --- a/salt/suricata/manager.sls +++ b/salt/suricata/manager.sls @@ -13,7 +13,7 @@ ruleslink: - name: /opt/so/saltstack/local/salt/suricata/rules - user: socore - group: socore - - target: /opt/so/rules/nids + - target: /opt/so/rules/nids/suri refresh_salt_master_fileserver_suricata_ruleslink: salt.runner: @@ -27,4 +27,4 @@ refresh_salt_master_fileserver_suricata_ruleslink: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From 2143881c0b560792e696cab50429b979822d1bac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 9 Feb 2024 10:22:25 -0500 Subject: [PATCH 35/63] specify *.rules --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 02c7c01e0..47d5addb5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -607,7 +607,7 @@ up_to_2.4.50() { echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules/nids echo "Moving Suricata rules" - mv -v /opt/so/rules/nids/* /opt/so/rules/nids/suri/. + mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/. INSTALLEDVERSION=2.4.50 } From 213ac822a849defadac85c47e91f9114266b95ca Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 9 Feb 2024 10:54:07 -0500 Subject: [PATCH 36/63] create dir and chown --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 47d5addb5..d93218db4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -607,6 +607,8 @@ up_to_2.4.50() { echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules/nids echo "Moving Suricata rules" + mkdir /opt/so/rules/nids/suri + chown socore:socore /opt/so/rules/nids/suri mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/. INSTALLEDVERSION=2.4.50 From 304ae49251b6dd99b03785d6ea5c48c0d8c5d63f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 9 Feb 2024 12:41:23 -0500 Subject: [PATCH 37/63] fix source --- salt/suricata/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index e0a157a85..3ec1324bf 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -89,7 +89,7 @@ suridatadir: surirulesync: file.recurse: - name: /opt/so/conf/suricata/rules/ - - source: salt://suricata/ + - source: salt://suri/ - user: 940 - group: 940 - show_changes: False From 66ac36a9440682efdea72d4018675b6252238f9c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 10 Feb 2024 11:07:26 -0500 Subject: [PATCH 38/63] Update soup --- salt/manager/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d93218db4..600cb5d4e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -610,6 +610,9 @@ up_to_2.4.50() { mkdir /opt/so/rules/nids/suri chown socore:socore /opt/so/rules/nids/suri mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/. + + echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml" + so-yaml.py append /etc/salt/master file_roots.base /nsm/elastic-fleet/artifacts INSTALLEDVERSION=2.4.50 } From eafb5cf15eed65c0ff2050b431ccceb0e0761f46 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 11 Feb 2024 13:18:20 -0500 Subject: [PATCH 39/63] Change to file_root --- salt/elasticfleet/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 53ec6c1ef..5b0cff5df 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -52,7 +52,7 @@ so-elastic-fleet-auto-configure-artifact-urls: elasticagent_syncartifacts: file.recurse: - name: /nsm/elastic-fleet/artifacts/beats - - source: salt://beats?saltenv=elasticartifacts + - source: salt://beats {% endif %} {% if SERVICETOKEN != '' %} From cc0f25a4f7f400759a7a3845c3a3fbb4a8bdc26e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 11 Feb 2024 13:30:20 -0500 Subject: [PATCH 40/63] Wait for ES to be ready --- salt/elasticfleet/enabled.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index fef85d24c..31c4e3469 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -17,6 +17,11 @@ include: - elasticfleet.sostatus - ssl +# Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready +wait_for_elasticsearch: + cmd.run: + - name: so-elasticsearch-wait + # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} so-elastic-fleet-auto-configure-logstash-outputs: From 4b697b24061b38f4004c9753c96f538d321164e4 Mon Sep 17 00:00:00 2001 From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com> Date: Mon, 12 Feb 2024 09:28:48 -0500 Subject: [PATCH 41/63] Remove unused file --- salt/stig/license.sls | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 salt/stig/license.sls diff --git a/salt/stig/license.sls b/salt/stig/license.sls deleted file mode 100644 index e69de29bb..000000000 From ccb14485a3028b326bb26f78b7341bd312fc1059 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Feb 2024 19:06:19 -0500 Subject: [PATCH 42/63] Fix conflicting id --- salt/elasticfleet/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 4fc738171..50290bf09 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -18,7 +18,7 @@ include: - ssl # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready -wait_for_elasticsearch: +wait_for_elasticsearch_elasticfleet: cmd.run: - name: so-elasticsearch-wait From 20d2f3b97e6170fb3216e6cfece9f9740582f5ba Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 12 Feb 2024 19:13:32 -0500 Subject: [PATCH 43/63] Update Sublime action in defaults.yaml to use i18n --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 31b6eb588..341bee64a 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -59,8 +59,8 @@ soc: target: _blank links: - 'https://www.virustotal.com/gui/search/{value}' - - name: Sublime Platform Email Review - description: Review email in Sublime Platform + - name: actionSublime + description: actionSublimeHelp icon: fa-external-link-alt target: _blank links: From 0ad39a7e327131c533958526fa3ed96f71565293 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 12 Feb 2024 19:18:29 -0500 Subject: [PATCH 44/63] FEATURE: Add new SOC action to show process ancestry #12345 --- salt/soc/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 341bee64a..7573854c6 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -65,6 +65,12 @@ soc: target: _blank links: - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - name: actionProcessAncestors + description: actionProcessAncestorsHelp + icon: fa-people-roof + target: '' + links: + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby event.module event.dataset | table soc_timestamp event.dataset user.name process.executable process.command_line process.working_directory' eventFields: default: - soc_timestamp From 3efaba11041df34e3bb74762241e013229c4238e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 11:04:26 -0500 Subject: [PATCH 45/63] modify soup to update soup scripts without using salt --- salt/common/init.sls | 13 ++++++++++++- salt/common/soup_scripts.sls | 23 ----------------------- salt/manager/tools/sbin/soup | 23 ++++++++--------------- 3 files changed, 20 insertions(+), 39 deletions(-) delete mode 100644 salt/common/soup_scripts.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index 5f13c3893..51836daf6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,7 +4,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} include: - - common.soup_scripts - common.packages {% if GLOBALS.role in GLOBALS.manager_roles %} - manager.elasticsearch # needed for elastic_curl_config state @@ -134,6 +133,18 @@ common_sbin_jinja: - file_mode: 755 - template: jinja +{% if not GLOBALS.is_manager%} +# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers +# these two states remove the scripts from non manager nodes +remove_soup: + file.absent: + - name: /usr/sbin/soup + +remove_so-firewall: + file.absent: + - name: /usr/sbin/so-firewall +{% endif %} + so-status_script: file.managed: - name: /usr/sbin/so-status diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls deleted file mode 100644 index 041649200..000000000 --- a/salt/common/soup_scripts.sls +++ /dev/null @@ -1,23 +0,0 @@ -# Sync some Utilities -soup_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://common/tools/sbin - - include_pat: - - so-common - - so-image-common - -soup_manager_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://manager/tools/sbin - - include_pat: - - so-firewall - - so-repo-sync - - soup diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 600cb5d4e..3254a61dd 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -794,21 +794,14 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - cp $UPDATE_DIR/salt/manager/tools/sbin/soup $DEFAULT_SALT_DIR/salt/manager/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/manager/tools/sbin/ - salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local - # Verify that soup scripts updated as expected - get_soup_script_hashes - if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then - echo "Succesfully updated soup scripts." - else - # When STIGs are enabled soup scripts will fail to update using --file-root --local. - # After checking that the expected hashes are not present, retry updating soup scripts using salt master. - echo "There was a problem updating soup scripts.. Trying to rerun script update" - salt-call state.apply common.soup_scripts queue=True -linfo - fi + rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup + rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall + + cp $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. + cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. + cp $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. + cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. + echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 From 92634724c40a97612fb254d0b84c068df2cc5742 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 11:09:08 -0500 Subject: [PATCH 46/63] move rm --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3254a61dd..02dd1272b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -794,14 +794,14 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup - rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall - cp $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. cp $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. + rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup + rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall + echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 From 8060751a667fe6d1b9d910a39ac5f851ac4d691e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 13 Feb 2024 12:24:33 -0500 Subject: [PATCH 47/63] Add table columns to process dashboard in defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7573854c6..ba557d64d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1417,7 +1417,7 @@ soc: query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint - query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp host.name user.name process.working_directory process.parent.name process.name' - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' From b713771494659b098f16c1d7b0eb37501721cf66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 12:30:36 -0500 Subject: [PATCH 48/63] add back common soup_scripts state --- salt/common/soup_scripts.sls | 24 ++++++++++++++++++++++++ salt/manager/tools/sbin/soup | 10 ++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 salt/common/soup_scripts.sls diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls new file mode 100644 index 000000000..402ad9c4a --- /dev/null +++ b/salt/common/soup_scripts.sls @@ -0,0 +1,24 @@ +remove_common_soup: + file.absent: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup + +remove_common_so-firewall: + file.absent: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall + +# Sync some Utilities +soup_scripts: + file.recurse: + - name: /usr/sbin + - user: root + - group: root + - file_mode: 755 + - source: salt://common/tools/sbin + +soup_manager_scripts: + file.recurse: + - name: /usr/sbin + - user: root + - group: root + - file_mode: 755 + - source: salt://manager/tools/sbin diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 02dd1272b..75be97928 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -795,12 +795,14 @@ verify_latest_update_script() { else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." cp $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. - cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. + #cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. cp $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. - cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. + #cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. - rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup - rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall + #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup + #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall + + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." From 5c9b1ab38b052e15f363daad0e6ffc43ddd8d8fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 12:48:31 -0500 Subject: [PATCH 49/63] copy with cp --- salt/common/soup_scripts.sls | 27 ++++++++++++--------------- salt/manager/tools/sbin/soup | 4 ++-- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 402ad9c4a..a4fafd6e3 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -6,19 +6,16 @@ remove_common_so-firewall: file.absent: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall -# Sync some Utilities -soup_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://common/tools/sbin +{% if pillar.global.airgap %} +{% set UPDATE_DIR='/tmp/soagupdate/securityonion'%} +{% else %} +{% set UPDATE_DIR='/tmp/sogh/securityonion'%} +{% endif %} -soup_manager_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://manager/tools/sbin +copy_common: + cmd.run: + - name: "\cp " ~ {{ $UPDATE_DIR }} ~ "/salt/common/tools/sbin/* /usr/sbin/." + +copy_manager: + cmd.run: + - name: "\cp " ~ {{ $UPDATE_DIR }} ~ "/salt/manager/tools/sbin/* /usr/sbin/." diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 75be97928..2dfad1bbb 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -795,9 +795,9 @@ verify_latest_update_script() { else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." cp $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. - #cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. + cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. cp $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. - #cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. + cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall From d7f853b5b2d48f30518b1629de03fa1bbcbd2d6d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 12:50:22 -0500 Subject: [PATCH 50/63] comment out script copy in soup --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2dfad1bbb..75be97928 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -795,9 +795,9 @@ verify_latest_update_script() { else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." cp $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. - cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. + #cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. cp $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. - cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. + #cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall From 0741ae370af217155b9075c865715d469e4edfda Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 13 Feb 2024 12:51:26 -0500 Subject: [PATCH 51/63] Update defaults.yaml --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ba557d64d..244a021d3 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -70,7 +70,7 @@ soc: icon: fa-people-roof target: '' links: - - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby event.module event.dataset | table soc_timestamp event.dataset user.name process.executable process.command_line process.working_directory' + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby event.module event.dataset | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' eventFields: default: - soc_timestamp @@ -1417,7 +1417,7 @@ soc: query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint - query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp host.name user.name process.working_directory process.parent.name process.name' + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' From 1bde002f20a1a6f58cb78c57be8a70fd94c7689d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 12:51:53 -0500 Subject: [PATCH 52/63] update case --- salt/common/soup_scripts.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index a4fafd6e3..346b63c96 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -7,7 +7,7 @@ remove_common_so-firewall: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall {% if pillar.global.airgap %} -{% set UPDATE_DIR='/tmp/soagupdate/securityonion'%} +{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion'%} {% else %} {% set UPDATE_DIR='/tmp/sogh/securityonion'%} {% endif %} From 9175a7345609a58602d20bc548f9b35f4b2bd497 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 13:08:09 -0500 Subject: [PATCH 53/63] dont need $ for vars --- salt/common/soup_scripts.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 346b63c96..4a4fe80c8 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -14,8 +14,8 @@ remove_common_so-firewall: copy_common: cmd.run: - - name: "\cp " ~ {{ $UPDATE_DIR }} ~ "/salt/common/tools/sbin/* /usr/sbin/." + - name: "\cp " ~ {{ UPDATE_DIR }} ~ "/salt/common/tools/sbin/* /usr/sbin/." copy_manager: cmd.run: - - name: "\cp " ~ {{ $UPDATE_DIR }} ~ "/salt/manager/tools/sbin/* /usr/sbin/." + - name: "\cp " ~ {{ UPDATE_DIR }} ~ "/salt/manager/tools/sbin/* /usr/sbin/." From d6ac7a32869c4c1ff84ab762eb261606576e66d2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 13:31:34 -0500 Subject: [PATCH 54/63] fix the jinja --- salt/common/soup_scripts.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 4a4fe80c8..86042a880 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -6,7 +6,7 @@ remove_common_so-firewall: file.absent: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall -{% if pillar.global.airgap %} +{% if salt['pillar.get']('global:airgap') %} {% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion'%} {% else %} {% set UPDATE_DIR='/tmp/sogh/securityonion'%} @@ -14,8 +14,8 @@ remove_common_so-firewall: copy_common: cmd.run: - - name: "\cp " ~ {{ UPDATE_DIR }} ~ "/salt/common/tools/sbin/* /usr/sbin/." + - name: "cp {{UPDATE_DIR}}/salt/common/tools/sbin/* /usr/sbin/." copy_manager: cmd.run: - - name: "\cp " ~ {{ UPDATE_DIR }} ~ "/salt/manager/tools/sbin/* /usr/sbin/." + - name: "cp {{UPDATE_DIR}}/salt/manager/tools/sbin/* /usr/sbin/." From 7112337c85ba0b75ac56d46da2965c0657a2cd66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 13:52:14 -0500 Subject: [PATCH 55/63] fix copy --- salt/manager/tools/sbin/soup | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 75be97928..b3df3eb6f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -794,13 +794,8 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - cp $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. - #cp $UPDATE_DIR/salt/common/tools/sbin/* /usr/sbin/. - cp $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. - #cp $UPDATE_DIR/salt/manager/tools/sbin/* /usr/sbin/. - - #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/soup - #rm -f $DEFAULT_SALT_DIR/salt/common/tools/sbin/so-firewall + \cp -v $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. + \cp -v $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local From 141fd49f02ec03a5639a6b2908c16d1ddbb81bd9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 14:27:22 -0500 Subject: [PATCH 56/63] use rsync --- salt/common/soup_scripts.sls | 16 ++++++++++++---- salt/manager/tools/sbin/soup | 2 -- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 86042a880..b4b3504d1 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -12,10 +12,18 @@ remove_common_so-firewall: {% set UPDATE_DIR='/tmp/sogh/securityonion'%} {% endif %} -copy_common: +copy_common_tools_sbin: cmd.run: - - name: "cp {{UPDATE_DIR}}/salt/common/tools/sbin/* /usr/sbin/." + - name: "rsync -avh {{UPDATE_DIR}}/salt/common/tools/sbin/* /opt/so/saltstack/default/salt/common/tools/sbin/" -copy_manager: +copy_manager_tools_sbin: cmd.run: - - name: "cp {{UPDATE_DIR}}/salt/manager/tools/sbin/* /usr/sbin/." + - name: "rsync -avh {{UPDATE_DIR}}/salt/manager/tools/sbin/* /opt/so/saltstack/default/salt/manager/tools/sbin/" + +copy_common_sbin: + cmd.run: + - name: "rsync -avh {{UPDATE_DIR}}/salt/common/tools/sbin/* /usr/sbin/" + +copy_manager_sbin: + cmd.run: + - name: "rsync -avh {{UPDATE_DIR}}/salt/manager/tools/sbin/* /usr/sbin/" diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b3df3eb6f..c4eef3994 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -794,8 +794,6 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - \cp -v $UPDATE_DIR/salt/common/tools/sbin/* $DEFAULT_SALT_DIR/salt/common/tools/sbin/. - \cp -v $UPDATE_DIR/salt/manager/tools/sbin/* $DEFAULT_SALT_DIR/salt/manager/tools/sbin/. salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local From 88786e83427ae49f236aa9f25c28f80684ec6845 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 15:05:09 -0500 Subject: [PATCH 57/63] use file.copy to preserve perms --- salt/common/soup_scripts.sls | 64 +++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index b4b3504d1..809886266 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -12,18 +12,58 @@ remove_common_so-firewall: {% set UPDATE_DIR='/tmp/sogh/securityonion'%} {% endif %} -copy_common_tools_sbin: - cmd.run: - - name: "rsync -avh {{UPDATE_DIR}}/salt/common/tools/sbin/* /opt/so/saltstack/default/salt/common/tools/sbin/" +copy_so-common_common_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common + - force: True + - preserve: True -copy_manager_tools_sbin: - cmd.run: - - name: "rsync -avh {{UPDATE_DIR}}/salt/manager/tools/sbin/* /opt/so/saltstack/default/salt/manager/tools/sbin/" +copy_so-image-common_common_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common + - force: True + - preserve: True -copy_common_sbin: - cmd.run: - - name: "rsync -avh {{UPDATE_DIR}}/salt/common/tools/sbin/* /usr/sbin/" +copy_soup_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup + - force: True + - preserve: True -copy_manager_sbin: - cmd.run: - - name: "rsync -avh {{UPDATE_DIR}}/salt/manager/tools/sbin/* /usr/sbin/" +copy_so-firewall_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall + - force: True + - preserve: True + +copy_so-common_sbin: + file.copy: + - name: /usr/sbin/so-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common + - force: True + - preserve: True + +copy_so-image-common_sbin: + file.copy: + - name: /usr/sbin/so-image-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common + - force: True + - preserve: True + +copy_soup_sbin: + file.copy: + - name: /usr/sbin/soup + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup + - force: True + - preserve: True + +copy_so-firewall_sbin: + file.copy: + - name: /usr/so-firewall + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall + - force: True + - preserve: True From 468eedfaeb7288d7ed3657da18856b297fe00f29 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 15:30:24 -0500 Subject: [PATCH 58/63] add soup script update retru --- salt/manager/tools/sbin/soup | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c4eef3994..0e982e51f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -797,6 +797,14 @@ verify_latest_update_script() { salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + get_soup_script_hashes + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then + echo "Succesfully updated soup scripts." + else + echo "There was a problem updating soup scripts. Trying to rerun script update." + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + fi + echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 From 00f2374582d915c02dab51e2710bd7fb6764caca Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 15:43:02 -0500 Subject: [PATCH 59/63] fix path for so-firewall --- salt/common/soup_scripts.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 809886266..c02f11172 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -63,7 +63,7 @@ copy_soup_sbin: copy_so-firewall_sbin: file.copy: - - name: /usr/so-firewall + - name: /usr/sbin/so-firewall - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall - force: True - preserve: True From 2e9fa2438b01eb65f16b942ea7fdcb6009486a9f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Feb 2024 16:19:50 -0500 Subject: [PATCH 60/63] add back comment --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0e982e51f..8b9d4a6b3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -797,6 +797,7 @@ verify_latest_update_script() { salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + # Verify that soup scripts updated as expected get_soup_script_hashes if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then echo "Succesfully updated soup scripts." From 79e98e508f0d6f0ebd100e4d38bf767151cc82a2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Feb 2024 13:28:12 -0500 Subject: [PATCH 61/63] pass in UPDATE_DIR as a pillar --- salt/common/soup_scripts.sls | 7 ++----- salt/manager/tools/sbin/soup | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index c02f11172..9c3b8ad3a 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -6,11 +6,8 @@ remove_common_so-firewall: file.absent: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall -{% if salt['pillar.get']('global:airgap') %} -{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion'%} -{% else %} -{% set UPDATE_DIR='/tmp/sogh/securityonion'%} -{% endif %} +{# this pillar isn't defined anywhere. it is passed in from soup when the state is called #} +{% set UPDATE_DIR= salt['pillar.get']('UPDATE_DIR') %} copy_so-common_common_tools_sbin: file.copy: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8b9d4a6b3..c44883d15 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -803,7 +803,7 @@ verify_latest_update_script() { echo "Succesfully updated soup scripts." else echo "There was a problem updating soup scripts. Trying to rerun script update." - salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local pillar="{'UPDATE_DIR': $UPDATE_DIR}" fi echo "" From c1f467a06819c22e3bec737c7270b11c31bc6d9f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Feb 2024 14:22:18 -0500 Subject: [PATCH 62/63] handle airgap --- salt/common/soup_scripts.sls | 8 ++++++-- salt/manager/tools/sbin/soup | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 9c3b8ad3a..5a7bdef47 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -6,8 +6,12 @@ remove_common_so-firewall: file.absent: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall -{# this pillar isn't defined anywhere. it is passed in from soup when the state is called #} -{% set UPDATE_DIR= salt['pillar.get']('UPDATE_DIR') %} +{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %} +{% if SOC_GLOBAL.global.airgap %} +{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %} +{% else %} +{% set UPDATE_DIR='/tmp/sogh/securityonion' %} +{% endif %} copy_so-common_common_tools_sbin: file.copy: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c44883d15..8b9d4a6b3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -803,7 +803,7 @@ verify_latest_update_script() { echo "Succesfully updated soup scripts." else echo "There was a problem updating soup scripts. Trying to rerun script update." - salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local pillar="{'UPDATE_DIR': $UPDATE_DIR}" + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local fi echo "" From a2b17d23485804bd4e4895f3e3e5f70d9a83d8e6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Feb 2024 14:27:41 -0500 Subject: [PATCH 63/63] move jinja to top --- salt/common/soup_scripts.sls | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 5a7bdef47..fd32b8a28 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -1,3 +1,10 @@ +{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %} +{% if SOC_GLOBAL.global.airgap %} +{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %} +{% else %} +{% set UPDATE_DIR='/tmp/sogh/securityonion' %} +{% endif %} + remove_common_soup: file.absent: - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup @@ -6,13 +13,6 @@ remove_common_so-firewall: file.absent: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall -{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %} -{% if SOC_GLOBAL.global.airgap %} -{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %} -{% else %} -{% set UPDATE_DIR='/tmp/sogh/securityonion' %} -{% endif %} - copy_so-common_common_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common