diff --git a/files/salt/master/master b/files/salt/master/master index b93fa93de..2333a50aa 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,7 +41,8 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt - + - /nsm/elastic-fleet/artifacts + - /opt/so/rules/nids # The master_roots setting configures a master-only copy of the file_roots dictionary, # used by the state compiler. diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index d27f51ede..3ead8b26e 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -180,6 +180,7 @@ 'telegraf', 'firewall', 'logstash', + 'nginx', 'healthcheck', 'schedule', 'elasticfleet', diff --git a/salt/common/init.sls b/salt/common/init.sls index 5f13c3893..51836daf6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,7 +4,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} include: - - common.soup_scripts - common.packages {% if GLOBALS.role in GLOBALS.manager_roles %} - manager.elasticsearch # needed for elastic_curl_config state @@ -134,6 +133,18 @@ common_sbin_jinja: - file_mode: 755 - template: jinja +{% if not GLOBALS.is_manager%} +# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers +# these two states remove the scripts from non manager nodes +remove_soup: + file.absent: + - name: /usr/sbin/soup + +remove_so-firewall: + file.absent: + - name: /usr/sbin/so-firewall +{% endif %} + so-status_script: file.managed: - name: /usr/sbin/so-status diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 041649200..fd32b8a28 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -1,23 +1,70 @@ -# Sync some Utilities -soup_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://common/tools/sbin - - include_pat: - - so-common - - so-image-common +{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %} +{% if SOC_GLOBAL.global.airgap %} +{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %} +{% else %} +{% set UPDATE_DIR='/tmp/sogh/securityonion' %} +{% endif %} -soup_manager_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://manager/tools/sbin - - include_pat: - - so-firewall - - so-repo-sync - - soup +remove_common_soup: + file.absent: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup + +remove_common_so-firewall: + file.absent: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall + +copy_so-common_common_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common + - force: True + - preserve: True + +copy_so-image-common_common_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common + - force: True + - preserve: True + +copy_soup_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup + - force: True + - preserve: True + +copy_so-firewall_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall + - force: True + - preserve: True + +copy_so-common_sbin: + file.copy: + - name: /usr/sbin/so-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common + - force: True + - preserve: True + +copy_so-image-common_sbin: + file.copy: + - name: /usr/sbin/so-image-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common + - force: True + - preserve: True + +copy_soup_sbin: + file.copy: + - name: /usr/sbin/soup + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup + - force: True + - preserve: True + +copy_so-firewall_sbin: + file.copy: + - name: /usr/sbin/so-firewall + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall + - force: True + - preserve: True diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 9a7d53317..b2a028e60 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -334,6 +334,7 @@ desktop_packages: - pulseaudio-libs - pulseaudio-libs-glib2 - pulseaudio-utils + - putty - sane-airscan - sane-backends - sane-backends-drivers-cameras diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 9a27843ae..4bc212fbe 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -84,6 +84,13 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] + 'so-nginx-fleet-node': + final_octet: 31 + port_bindings: + - 8443:8443 + custom_bind_mounts: [] + extra_hosts: [] + extra_env: [] 'so-playbook': final_octet: 32 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index 850324a9e..6e0efeb20 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -48,6 +48,7 @@ docker: so-logstash: *dockerOptions so-mysql: *dockerOptions so-nginx: *dockerOptions + so-nginx-fleet-node: *dockerOptions so-playbook: *dockerOptions so-redis: *dockerOptions so-sensoroni: *dockerOptions diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index fef85d24c..50290bf09 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -17,6 +17,11 @@ include: - elasticfleet.sostatus - ssl +# Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready +wait_for_elasticsearch_elasticfleet: + cmd.run: + - name: so-elasticsearch-wait + # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} so-elastic-fleet-auto-configure-logstash-outputs: @@ -33,12 +38,26 @@ so-elastic-fleet-auto-configure-server-urls: - retry: True {% endif %} -# Automatically update Fleet Server Elasticsearch URLs +# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs {% if grains.role not in ['so-fleet'] %} so-elastic-fleet-auto-configure-elasticsearch-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-es-url-update - retry: True + +so-elastic-fleet-auto-configure-artifact-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-artifacts-url-update + - retry: True + +{% endif %} + +# Sync Elastic Agent artifacts to Fleet Node +{% if grains.role in ['so-fleet'] %} +elasticagent_syncartifacts: + file.recurse: + - name: /nsm/elastic-fleet/artifacts/beats + - source: salt://beats {% endif %} {% if SERVICETOKEN != '' %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index b1ca8c476..b911f5896 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update new file mode 100644 index 000000000..721525668 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -0,0 +1,90 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +# Function to check if an array contains a value +array_contains () { + local array="$1[@]" + local seeking=$2 + local in=1 + for element in "${!array}"; do + if [[ $element == "$seeking" ]]; then + in=0 + break + fi + done + return $in +} + +# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) +LOGSTASHNODES='{{ salt['pillar.get']('logstash:nodes', {}) | tojson }}' + +# Initialize an array for new hosts from Fleet Nodes +declare -a NEW_LIST=() + +# Query for Fleet Nodes & add them to the list (Hostname) +if grep -q "fleet" <<< "$LOGSTASHNODES"; then + readarray -t FLEETNODES < <(jq -r '.fleet | keys_unsorted[]' <<< "$LOGSTASHNODES") + for NODE in "${FLEETNODES[@]}"; do + URL="http://$NODE:8443/artifacts/" + NAME="FleetServer_$NODE" + NEW_LIST+=("$URL=$NAME") + done +fi + +# Create an array for expected hosts and their names +declare -A expected_urls=( + ["http://{{ GLOBALS.url_base }}:8443/artifacts/"]="FleetServer_{{ GLOBALS.hostname }}" + ["https://artifacts.elastic.co/downloads/"]="Elastic Artifacts" +) + +# Merge NEW_LIST into expected_urls +for entry in "${NEW_LIST[@]}"; do + # Extract URL and Name from each entry + IFS='=' read -r URL NAME <<< "$entry" + # Add to expected_urls, automatically handling URL as key and NAME as value + expected_urls["$URL"]="$NAME" +done + +# Fetch the current hosts from the API +current_urls=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host) + +# Convert current hosts to an array +IFS=$'\n' read -rd '' -a current_urls_array <<<"$current_urls" + +# Flag to track if any host was added +any_url_added=0 + +# Check each expected host +for host in "${!expected_urls[@]}"; do + array_contains current_urls_array "$host" || { + echo "$host (${expected_urls[$host]}) is missing. Adding it..." + + # Prepare the JSON payload + JSON_STRING=$( jq -n \ + --arg NAME "${expected_urls[$host]}" \ + --arg URL "$host" \ + '{"name":$NAME,"host":$URL}' ) + + # Create the missing host + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + + # Flag that an artifact URL was added + any_url_added=1 + } + +done + + +if [[ $any_url_added -eq 0 ]]; then + echo "All expected artifact URLs are present. No updates needed." +fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 5d5b7e7e0..3da6b3e78 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 23a206921..eb5ccc1ed 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 31c7becca..5f7637cd3 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 23e11a710..27a8a0fd6 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,19 @@ esingestconf: - user: 930 - group: 939 +# Auto-generate Elasticsearch ingest node pipelines from pillar +{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} +es_ingest_conf_{{pipeline}}: + file.managed: + - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }} + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ config }} + - template: jinja + - onchanges_in: + - file: so-pipelines-reload +{% endfor %} + eslog4jfile: file.managed: - name: /opt/so/conf/elasticsearch/log4j2.properties diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 480467129..f4d8c8a95 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -55,6 +55,87 @@ elasticsearch: key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none enabled: false + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common index_settings: global_overrides: index_template: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index c54e07660..9a64190b3 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -45,6 +45,28 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html + pipelines: + custom001: &pipelines + description: + description: Description of the ingest node pipeline + global: True + advanced: True + helpLink: elasticsearch.html + processors: + description: Processors for the ingest node pipeline + global: True + advanced: True + multiline: True + helpLink: elasticsearch.html + custom002: *pipelines + custom003: *pipelines + custom004: *pipelines + custom005: *pipelines + custom006: *pipelines + custom007: *pipelines + custom008: *pipelines + custom009: *pipelines + custom010: *pipelines index_settings: global_overrides: index_template: diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 0ba2389e9..b3ead0f4c 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -95,6 +95,7 @@ {% set NODE_CONTAINERS = [ 'so-elastic-fleet', 'so-logstash', + 'so-nginx-fleet-node' ] %} {% elif GLOBALS.role == 'so-sensor' %} diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index decc5a5b2..5e4c4c066 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -39,7 +39,7 @@ so-idstools: {% endif %} - binds: - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - - /opt/so/rules/nids:/opt/so/rules/nids:rw + - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw - /nsm/rules/:/nsm/rules/:rw {% if DOCKER.containers['so-idstools'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %} diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf index d6f3d93d8..f7c784413 100644 --- a/salt/idstools/etc/rulecat.conf +++ b/salt/idstools/etc/rulecat.conf @@ -1,10 +1,10 @@ {%- from 'vars/globals.map.jinja' import GLOBALS -%} {%- from 'idstools/map.jinja' import IDSTOOLSMERGED -%} ---merged=/opt/so/rules/nids/all.rules ---local=/opt/so/rules/nids/local.rules +--merged=/opt/so/rules/nids/suri/all.rules +--local=/opt/so/rules/nids/suri/local.rules {%- if GLOBALS.md_engine == "SURICATA" %} ---local=/opt/so/rules/nids/extraction.rules ---local=/opt/so/rules/nids/filters.rules +--local=/opt/so/rules/nids/suri/extraction.rules +--local=/opt/so/rules/nids/suri/filters.rules {%- endif %} --url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules --disable=/opt/so/idstools/etc/disable.conf diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index 64479e937..cdacfaa74 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -21,7 +21,7 @@ idstoolsetcsync: rulesdir: file.directory: - - name: /opt/so/rules/nids + - name: /opt/so/rules/nids/suri - user: 939 - group: 939 - makedirs: True @@ -29,7 +29,7 @@ rulesdir: # Don't show changes because all.rules can be large synclocalnidsrules: file.recurse: - - name: /opt/so/rules/nids/ + - name: /opt/so/rules/nids/suri/ - source: salt://idstools/rules/ - user: 939 - group: 939 diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 6ff33ff1a..8a59c83b7 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -63,6 +63,20 @@ lspipelinedir: - user: 931 - group: 939 +# Auto-generate Logstash pipeline config +{% for pipeline, config in LOGSTASH_MERGED.pipeline_config.items() %} +{% for assigned_pipeline in ASSIGNED_PIPELINES %} +{% set custom_pipeline = 'custom/' + pipeline + '.conf' %} +{% if custom_pipeline in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: + file.managed: + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{ pipeline }}.conf + - contents: LOGSTASH_MERGED.pipeline_config.{{pipeline}} +{% endif %} +{% endfor %} +{% endfor %} + + {% for assigned_pipeline in ASSIGNED_PIPELINES %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index e4c18cc64..348acb622 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -42,6 +42,24 @@ logstash: custom2: [] custom3: [] custom4: [] + pipeline_config: + custom001: |- + filter { + if [event][module] =~ "zeek" { + mutate { + add_tag => ["network_stuff"] + } + } + } + custom002: PLACEHOLDER + custom003: PLACEHOLDER + custom004: PLACEHOLDER + custom005: PLACEHOLDER + custom006: PLACEHOLDER + custom007: PLACEHOLDER + custom008: PLACEHOLDER + custom009: PLACEHOLDER + custom010: PLACEHOLDER settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index bcb99bad5..3172ff7c5 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -31,6 +31,22 @@ logstash: custom2: *defined_pipelines custom3: *defined_pipelines custom4: *defined_pipelines + pipeline_config: + custom001: &pipeline_config + description: Pipeline configuration for Logstash + advanced: True + multiline: True + forcedType: string + helpLink: logstash.html + custom002: *pipeline_config + custom003: *pipeline_config + custom004: *pipeline_config + custom005: *pipeline_config + custom006: *pipeline_config + custom007: *pipeline_config + custom008: *pipeline_config + custom009: *pipeline_config + custom010: *pipeline_config settings: lsheap: description: Heap size to use for logstash diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 874fc9e0f..41cab0b23 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -16,12 +16,14 @@ lockFile = "/tmp/so-yaml.lock" def showUsage(args): print('Usage: {} [ARGS...]'.format(sys.argv[0])) print(' General commands:') + print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.') print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') print(' help - Prints this usage information.') print('') print(' Where:') print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') + print(' LISTITEM - Item to add to the list.') sys.exit(1) @@ -35,6 +37,35 @@ def writeYaml(filename, content): file = open(filename, "w") return yaml.dump(content, file) +def appendItem(content, key, listItem): + pieces = key.split(".", 1) + if len(pieces) > 1: + appendItem(content[pieces[0]], pieces[1], listItem) + else: + try: + content[key].append(listItem) + except AttributeError: + print("The existing value for the given key is not a list. No action was taken on the file.") + return 1 + except KeyError: + print("The key provided does not exist. No action was taken on the file.") + return 1 + +def append(args): + if len(args) != 3: + print('Missing filename, key arg, or list item to append', file=sys.stderr) + showUsage(None) + return + + filename = args[0] + key = args[1] + listItem = args[2] + + content = loadYaml(filename) + appendItem(content, key, listItem) + writeYaml(filename, content) + + return 0 def removeKey(content, key): pieces = key.split(".", 1) @@ -69,6 +100,7 @@ def main(): commands = { "help": showUsage, + "append": append, "remove": remove, } diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7d0ed1a8e..488877ea1 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -105,3 +105,99 @@ class TestRemove(unittest.TestCase): self.assertEqual(actual, expected) sysmock.assert_called_once_with(1) self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + + def test_append(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}") + file.close() + + soyaml.append([filename, "key3", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + expected = "key1:\n child1: 123\n child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\n- d\n" + self.assertEqual(actual, expected) + + def test_append_nested(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_append_nested_deep(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2.deep2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_append_key_noexist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key4", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_noexist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_nonlist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") + + def test_append_key_nonlist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a250116d1..8b9d4a6b3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -594,6 +594,26 @@ up_to_2.4.50() { touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls + # the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots + # put new so-yaml in place + echo "Updating so-yaml" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ + echo "Creating a backup of the salt-master config." + # INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version + # so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until + # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup + cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak" + echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" + so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules/nids + echo "Moving Suricata rules" + mkdir /opt/so/rules/nids/suri + chown socore:socore /opt/so/rules/nids/suri + mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/. + + echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml" + so-yaml.py append /etc/salt/master file_roots.base /nsm/elastic-fleet/artifacts + INSTALLEDVERSION=2.4.50 } @@ -774,21 +794,18 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - cp $UPDATE_DIR/salt/manager/tools/sbin/soup $DEFAULT_SALT_DIR/salt/manager/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/manager/tools/sbin/ + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + # Verify that soup scripts updated as expected get_soup_script_hashes if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then echo "Succesfully updated soup scripts." else - # When STIGs are enabled soup scripts will fail to update using --file-root --local. - # After checking that the expected hashes are not present, retry updating soup scripts using salt master. - echo "There was a problem updating soup scripts.. Trying to rerun script update" - salt-call state.apply common.soup_scripts queue=True -linfo + echo "There was a problem updating soup scripts. Trying to rerun script update." + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local fi + echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 @@ -937,9 +954,6 @@ main() { systemctl_func "stop" "$cron_service_name" - # update mine items prior to stopping salt-minion and salt-master - update_salt_mine - echo "Updating dockers to $NEWVERSION." if [[ $is_airgap -eq 0 ]]; then airgap_update_dockers @@ -1015,6 +1029,9 @@ main() { salt-call state.apply salt.minion -l info queue=True echo "" + # ensure the mine is updated and populated before highstates run, following the salt-master restart + update_salt_mine + enable_highstate echo "" diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index dda475655..273fb65be 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -14,6 +14,9 @@ include: - nginx.config - nginx.sostatus + +{% if grains.role not in ['so-fleet'] %} + {# if the user has selected to replace the crt and key in the ui #} {% if NGINXMERGED.ssl.replace_cert %} @@ -88,6 +91,15 @@ make-rule-dir-nginx: - recurse: - user - group + +{% endif %} + +{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #} +{% if GLOBALS.role == 'so-fleet' %} +{% set container_config = 'so-nginx-fleet-node' %} +{% else %} +{% set container_config = 'so-nginx' %} +{% endif %} so-nginx: docker_container.running: @@ -95,11 +107,11 @@ so-nginx: - hostname: so-nginx - networks: - sobridge: - - ipv4_address: {{ DOCKER.containers['so-nginx'].ip }} + - ipv4_address: {{ DOCKER.containers[container_config].ip }} - extra_hosts: - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} - {% if DOCKER.containers['so-nginx'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-nginx'].extra_hosts %} + {% if DOCKER.containers[container_config].extra_hosts %} + {% for XTRAHOST in DOCKER.containers[container_config].extra_hosts %} - {{ XTRAHOST }} {% endfor %} {% endif %} @@ -119,20 +131,20 @@ so-nginx: - /nsm/repo:/opt/socore/html/repo:ro - /nsm/rules:/nsm/rules:ro {% endif %} - {% if DOCKER.containers['so-nginx'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-nginx'].custom_bind_mounts %} + {% if DOCKER.containers[container_config].custom_bind_mounts %} + {% for BIND in DOCKER.containers[container_config].custom_bind_mounts %} - {{ BIND }} {% endfor %} {% endif %} - {% if DOCKER.containers['so-nginx'].extra_env %} + {% if DOCKER.containers[container_config].extra_env %} - environment: - {% for XTRAENV in DOCKER.containers['so-nginx'].extra_env %} + {% for XTRAENV in DOCKER.containers[container_config].extra_env %} - {{ XTRAENV }} {% endfor %} {% endif %} - cap_add: NET_BIND_SERVICE - port_bindings: - {% for BINDING in DOCKER.containers['so-nginx'].port_bindings %} + {% for BINDING in DOCKER.containers[container_config].port_bindings %} - {{ BINDING }} {% endfor %} - watch: diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index d5981be77..236f8da7f 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -39,6 +39,26 @@ http { include /etc/nginx/conf.d/*.conf; + {%- if role in ['fleet'] %} + + server { + listen 8443; + server_name {{ GLOBALS.hostname }}; + root /opt/socore/html; + location /artifacts/ { + try_files $uri =206; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + + {%- endif %} + {%- if role in ['eval', 'managersearch', 'manager', 'standalone', 'import'] %} server { diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 1b4d2e63a..19677f70b 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.5 + version: 3006.6 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index c15929951..2e4ebc93e 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.5 + version: 3006.6 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c060698b4..a19fb45f0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -9,7 +9,7 @@ soc: icon: fa-crosshairs target: links: - - '/#/hunt?q="{value|escape}" | groupby event.module* event.dataset' + - '/#/hunt?q="{value|escape}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - name: actionAddToCase description: actionAddToCaseHelp icon: fa-briefcase @@ -23,13 +23,13 @@ soc: icon: fab fa-searchengin target: '' links: - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* event.dataset' - - '/#/hunt?q="{:log.id.uid}" | groupby event.module* event.dataset' - - '/#/hunt?q="{:network.community_id}" | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:log.id.uid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:network.community_id}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - name: actionPcap description: actionPcapHelp icon: fa-stream @@ -59,12 +59,18 @@ soc: target: _blank links: - 'https://www.virustotal.com/gui/search/{value}' - - name: Sublime Platform Email Review - description: Review email in Sublime Platform + - name: actionSublime + description: actionSublimeHelp icon: fa-external-link-alt target: _blank links: - - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - name: actionProcessAncestors + description: actionProcessAncestorsHelp + icon: fa-people-roof + target: '' + links: + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby event.module event.dataset | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' eventFields: default: - soc_timestamp @@ -1425,7 +1431,7 @@ soc: query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint - query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' @@ -1438,8 +1444,11 @@ soc: - name: Zeek Notice description: Zeek notice logs query: 'event.dataset:zeek.notice | groupby -sankey notice.note destination.ip | groupby notice.note | groupby notice.message | groupby notice.sub_message | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - - name: Connections - description: Network connection metadata + - name: Connections and Metadata with community_id + description: Network connections that include community_id + query: '_exists_:network.community_id | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - name: Connections seen by Zeek or Suricata + description: Network connections logged by Zeek or Suricata query: 'tags:conn | groupby source.ip | groupby destination.ip | groupby destination.port | groupby -sankey destination.port network.protocol | groupby network.protocol | groupby network.transport | groupby connection.history | groupby connection.state | groupby connection.state_description | groupby source.geo.country_name | groupby destination.geo.country_name | groupby client.ip_bytes | groupby server.ip_bytes | groupby client.oui' - name: DCE_RPC description: DCE_RPC (Distributed Computing Environment / Remote Procedure Calls) network metadata @@ -1576,6 +1585,9 @@ soc: - name: Firewall description: Firewall logs query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Firewall Auth + description: Firewall authentication logs + query: 'observer.type:firewall AND event.category:authentication | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | table soc_timestamp user.name source.ip message' - name: VLAN description: VLAN (Virtual Local Area Network) tagged logs query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name' diff --git a/salt/ssl/remove.sls b/salt/ssl/remove.sls index 43a245288..7738490e5 100644 --- a/salt/ssl/remove.sls +++ b/salt/ssl/remove.sls @@ -2,6 +2,10 @@ trusttheca: file.absent: - name: /etc/pki/tls/certs/intca.crt +symlinkca: + file.absent: + - name: /etc/ssl/certs/intca.crt + influxdb_key: file.absent: - name: /etc/pki/influxdb.key diff --git a/salt/stig/license.sls b/salt/stig/license.sls deleted file mode 100644 index e69de29bb..000000000 diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 8d5279349..3ec1324bf 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -84,10 +84,12 @@ suridatadir: - mode: 770 - makedirs: True +# salt:// would resolve to /opt/so/rules/nids because of the defined file_roots and +# not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt surirulesync: file.recurse: - name: /opt/so/conf/suricata/rules/ - - source: salt://suricata/rules/ + - source: salt://suri/ - user: 940 - group: 940 - show_changes: False diff --git a/salt/suricata/manager.sls b/salt/suricata/manager.sls index c196c5cae..3d5183556 100644 --- a/salt/suricata/manager.sls +++ b/salt/suricata/manager.sls @@ -13,7 +13,7 @@ ruleslink: - name: /opt/so/saltstack/local/salt/suricata/rules - user: socore - group: socore - - target: /opt/so/rules/nids + - target: /opt/so/rules/nids/suri refresh_salt_master_fileserver_suricata_ruleslink: salt.runner: @@ -27,4 +27,4 @@ refresh_salt_master_fileserver_suricata_ruleslink: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/top.sls b/salt/top.sls index f8979956e..16b355476 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -264,6 +264,7 @@ base: - telegraf - firewall - logstash + - nginx - elasticfleet - elasticfleet.install_agent_grid - schedule diff --git a/setup/so-functions b/setup/so-functions index 46bb0b4b2..f0462e4d6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1600,6 +1600,9 @@ reinstall_init() { salt-call -l info saltutil.kill_all_jobs --local fi + logCmd "salt-call state.apply ca.remove -linfo --local --file-root=../salt" + logCmd "salt-call state.apply ssl.remove -linfo --local --file-root=../salt" + # Kill any salt processes (safely) for service in "${salt_services[@]}"; do # Stop the service in the background so we can exit after a certain amount of time @@ -1621,9 +1624,6 @@ reinstall_init() { done done - logCmd "salt-call state.apply ca.remove -linfo --local --file-root=../salt" - logCmd "salt-call state.apply ssl.remove -linfo --local --file-root=../salt" - # Remove all salt configs rm -rf /etc/salt/engines/* /etc/salt/grains /etc/salt/master /etc/salt/master.d/* /etc/salt/minion /etc/salt/minion.d/* /etc/salt/pki/* /etc/salt/proxy /etc/salt/proxy.d/* /var/cache/salt/ @@ -2152,11 +2152,12 @@ set_default_log_size() { esac local disk_dir="/" - if [ -d /nsm ]; then + if mountpoint -q /nsm; then disk_dir="/nsm" fi - if [ -d /nsm/elasticsearch ]; then + if mountpoint -q /nsm/elasticsearch; then disk_dir="/nsm/elasticsearch" + percentage=80 fi local disk_size_1k