diff --git a/.github/DISCUSSION_TEMPLATE/2-4.yml b/.github/DISCUSSION_TEMPLATE/2-4.yml index 3ba6cbd94..fbd4d2c22 100644 --- a/.github/DISCUSSION_TEMPLATE/2-4.yml +++ b/.github/DISCUSSION_TEMPLATE/2-4.yml @@ -26,6 +26,8 @@ body: - 2.4.120 - 2.4.130 - 2.4.140 + - 2.4.141 + - 2.4.150 - Other (please provide detail below) validations: required: true diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 018b02890..49f2b32b9 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -1,10 +1,6 @@ name: python-test on: - push: - paths: - - "salt/sensoroni/files/analyzers/**" - - "salt/manager/tools/sbin" pull_request: paths: - "salt/sensoroni/files/analyzers/**" @@ -17,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + python-version: ["3.13"] python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"] steps: @@ -36,4 +32,4 @@ jobs: flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12 --doctests --max-line-length=200 --statistics - name: Test with pytest run: | - pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini + PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 6b50d3190..c72865113 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.141-20250331 ISO image released on 2025/03/31 +### 2.4.150-20250512 ISO image released on 2025/05/12 ### Download and Verify -2.4.141-20250331 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.141-20250331.iso +2.4.150-20250512 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250512.iso -MD5: CAE347BC0437A93DC8F4089973ED0EA7 -SHA1: 3A6F0C2F3B6E3625E06F67EB251372D7E592CB0E -SHA256: D0426D8E55E01A0FBA15AFE0BB7887CCB724C07FE82DA706CD1592E6001CD12B +MD5: 7A7469A7A38EA9A2DB770C36AE36A0CA +SHA1: 7E768D515353F339DC536DED6207B786DAFF7D27 +SHA256: F8B2EB6B332F2367F0C097D211577565C8FB5CC7809E97D63687C634035B3699 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.141-20250331.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250512.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.141-20250331.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250512.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.141-20250331.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250512.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.141-20250331.iso.sig securityonion-2.4.141-20250331.iso +gpg --verify securityonion-2.4.150-20250512.iso.sig securityonion-2.4.150-20250512.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Fri 28 Mar 2025 06:28:11 PM EDT using RSA key ID FE507013 +gpg: Signature made Fri 09 May 2025 06:27:29 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/VERSION b/VERSION index 7b4caac73..316ab4cee 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.141 +2.4.150 diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls index 5801d36f1..a2528a476 100644 --- a/pillar/node_data/ips.sls +++ b/pillar/node_data/ips.sls @@ -24,6 +24,7 @@ {% endif %} {% endfor %} +{% if node_types %} node_data: {% for node_type, host_values in node_types.items() %} {% for hostname, details in host_values.items() %} @@ -33,3 +34,6 @@ node_data: role: {{node_type}} {% endfor %} {% endfor %} +{% else %} +node_data: False +{% endif %} diff --git a/pillar/top.sls b/pillar/top.sls index b8d694e23..33b5feb2d 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -24,10 +24,10 @@ base: - firewall.adv_firewall - nginx.soc_nginx - nginx.adv_nginx - - node_data.ips '*_manager or *_managersearch': - match: compound + - node_data.ips {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -90,6 +90,7 @@ base: - soc.license '*_eval': + - node_data.ips - secrets - healthcheck.eval - elasticsearch.index_templates @@ -138,6 +139,7 @@ base: - minions.adv_{{ grains.id }} '*_standalone': + - node_data.ips - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash @@ -260,6 +262,7 @@ base: - soc.license '*_import': + - node_data.ips - secrets - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} @@ -305,6 +308,7 @@ base: - minions.adv_{{ grains.id }} '*_fleet': + - node_data.ips - backup.soc_backup - backup.adv_backup - logstash.nodes diff --git a/salt/common/init.sls b/salt/common/init.sls index d4d90cbed..55f0d2031 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -129,6 +129,10 @@ common_sbin: - group: 939 - file_mode: 755 - show_changes: False +{% if GLOBALS.role == 'so-heavynode' %} + - exclude_pat: + - so-pcap-import +{% endif %} common_sbin_jinja: file.recurse: @@ -139,6 +143,20 @@ common_sbin_jinja: - file_mode: 755 - template: jinja - show_changes: False +{% if GLOBALS.role == 'so-heavynode' %} + - exclude_pat: + - so-import-pcap +{% endif %} + +{% if GLOBALS.role == 'so-heavynode' %} +remove_so-pcap-import_heavynode: + file.absent: + - name: /usr/sbin/so-pcap-import + +remove_so-import-pcap_heavynode: + file.absent: + - name: /usr/sbin/so-import-pcap +{% endif %} {% if not GLOBALS.is_manager%} # prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 66f6825e9..96881aa56 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -128,6 +128,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available" # Typical error when making a query before ES has finished loading all indices EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP" # SO does not bundle the maxminddb with Zeek fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 7c776937d..2d7ad4e1c 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -200,6 +200,7 @@ docker: final_octet: 88 port_bindings: - 0.0.0.0:9092:9092 + - 0.0.0.0:29092:29092 - 0.0.0.0:9093:9093 - 0.0.0.0:8778:8778 custom_bind_mounts: [] diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a0f509136..d6cdd7351 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -11,6 +11,7 @@ elasticfleet: defend_filters: enable_auto_configuration: False subscription_integrations: False + auto_upgrade_integrations: False logging: zeek: excluded: diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json new file mode 100644 index 000000000..f6b01cdff --- /dev/null +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -0,0 +1,46 @@ +{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%} +{%- set valid_identities = false -%} +{%- if identities -%} + {%- set valid_identities = true -%} + {%- for id, email in identities -%} + {%- if not id or not email -%} + {%- set valid_identities = false -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} +{%- endif -%} +{ + "package": { + "name": "log", + "version": "" + }, + "name": "kratos-logs", + "namespace": "so", + "description": "Kratos logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.logs": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/kratos/kratos.log" + ], + "data_stream.dataset": "kratos", + "tags": ["so-kratos"], + {%- if valid_identities -%} + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos\n- if:\n has_fields:\n - identity_id\n then:{% for id, email in identities %}\n - if:\n equals:\n identity_id: \"{{ id }}\"\n then:\n - add_fields:\n target: ''\n fields:\n user.name: \"{{ email }}\"{% endfor %}", + {%- else -%} + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", + {%- endif -%} + "custom": "pipeline: kratos" + } + } + } + } + }, + "force": true +} + diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json deleted file mode 100644 index 6a67c9c1c..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "kratos-logs", - "namespace": "so", - "description": "Kratos logs", - "policy_id": "so-grid-nodes_general", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.logs": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/kratos/kratos.log" - ], - "data_stream.dataset": "kratos", - "tags": ["so-kratos"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", - "custom": "pipeline: kratos" - } - } - } - } - }, - "force": true -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json index 98204e894..4ee4d2b65 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -31,7 +31,8 @@ ], "tags": [ "so-grid-node" - ] + ], + "processors": "- if:\n contains:\n message: \"salt-minion\"\n then: \n - dissect:\n tokenizer: \"%{} %{} %{} %{} %{} %{}: [%{log.level}] %{*}\"\n field: \"message\"\n trim_values: \"all\"\n target_prefix: \"\"\n - drop_event:\n when:\n equals:\n log.level: \"INFO\"" } } } diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 7ca59401f..450e044e6 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -45,6 +45,11 @@ elasticfleet: global: True forcedType: bool helpLink: elastic-fleet.html + auto_upgrade_integrations: + description: Enables or disables automatically upgrading Elastic Agent integrations. + global: True + forcedType: bool + helpLink: elastic-fleet.html server: custom_fqdn: description: Custom FQDN for Agents to connect to. One per line. diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade deleted file mode 100644 index baad389eb..000000000 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -. /usr/sbin/so-elastic-fleet-common - -curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/) -if [ $? -ne 0 ]; then - echo "Error: Failed to connect to Kibana." - exit 1 -fi - -IFS=$'\n' -agent_policies=$(elastic_fleet_agent_policy_ids) -if [ $? -ne 0 ]; then - echo "Error: Failed to retrieve agent policies." - exit 1 -fi - -for AGENT_POLICY in $agent_policies; do - integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") - for INTEGRATION in $integrations; do - if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then - # Get package name so we know what package to look for when checking the current and latest available version - PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") - - # Get currently installed version of package - PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") - - # Get latest available version of package - AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME") - - # Get integration ID - INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION") - - if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then - # Dry run of the upgrade - echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." - echo "Upgrading $INTEGRATION..." - echo "Starting dry run..." - DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID") - DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) - - # If no errors with dry run, proceed with actual upgrade - if [[ "$DRYRUN_ERRORS" == "false" ]]; then - echo "No errors detected. Proceeding with upgrade..." - elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" - if [ $? -ne 0 ]; then - echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'." - exit 1 - fi - else - echo "Errors detected during dry run. Stopping upgrade..." - exit 1 - fi - fi - fi - done -done -echo diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 673fe6f2e..d25c18e29 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -83,5 +83,10 @@ docker run \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" -printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n" +printf "\n### Cleaning up temp files \n" rm -rf /nsm/elastic-agent-workspace +rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe + +printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n" +\cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/ +chmod 644 /nsm/elastic-fleet/so_agent-installers/* diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index 1ce379c1c..449d26c99 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -14,7 +14,7 @@ if ! is_manager_node; then fi # Get current list of Grid Node Agents that need to be upgraded -RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true") +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%20:%20%22{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%22%20and%20policy_id%20:%20%22so-grid-nodes_general%22&showInactive=false&getStatusSummary=true") # Check to make sure that the server responded with good data - else, bail from script CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON") diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade new file mode 100644 index 000000000..54540ba33 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} +{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %} + +. /usr/sbin/so-elastic-fleet-common + +curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/) +if [ $? -ne 0 ]; then + echo "Error: Failed to connect to Kibana." + exit 1 +fi + +IFS=$'\n' +agent_policies=$(elastic_fleet_agent_policy_ids) +if [ $? -ne 0 ]; then + echo "Error: Failed to retrieve agent policies." + exit 1 +fi + +default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + +for AGENT_POLICY in $agent_policies; do + integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + for INTEGRATION in $integrations; do + if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then + # Get package name so we know what package to look for when checking the current and latest available version + PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then + {%- endif %} + # Get currently installed version of package + PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") + + # Get latest available version of package + AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME") + + # Get integration ID + INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION") + + if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then + # Dry run of the upgrade + echo "" + echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." + echo "Upgrading $INTEGRATION..." + echo "Starting dry run..." + DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID") + DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) + + # If no errors with dry run, proceed with actual upgrade + if [[ "$DRYRUN_ERRORS" == "false" ]]; then + echo "No errors detected. Proceeding with upgrade..." + elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" + if [ $? -ne 0 ]; then + echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'." + exit 1 + fi + else + echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..." + exit 1 + fi + fi + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + fi + {%- endif %} + fi + done +done +echo diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index f97ed577b..26d775e82 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -3,7 +3,10 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %} +{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common @@ -46,6 +49,28 @@ compare_versions() { fi } +IFS=$'\n' +agent_policies=$(elastic_fleet_agent_policy_ids) +if [ $? -ne 0 ]; then + echo "Error: Failed to retrieve agent policies." + exit 1 +fi + +default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + +in_use_integrations=() + +for AGENT_POLICY in $agent_policies; do + integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + for INTEGRATION in $integrations; do + PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + # non-default integrations that are in-use in any policy + if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then + in_use_integrations+=("$PACKAGE_NAME") + fi + done +done + if [[ -f $STATE_FILE_SUCCESS ]]; then if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then # Package_list contains all integrations beta / non-beta. @@ -77,10 +102,19 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then + {%- endif %} + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true + PENDING_UPDATE=true + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + else + echo "skipping available upgrade for in use integration - $package_name." + fi + {%- endif %} fi fi fi @@ -92,9 +126,18 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true + {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then + {%- endif %} + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + else + echo "skipping available upgrade for in use integration - $package_name." + fi + {%- endif %} fi fi {% endif %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy index 7727f7db1..a5ea79922 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy +++ b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy @@ -32,7 +32,7 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then --arg KAFKACA "$KAFKACA" \ --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \ --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \ - '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }' + '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 10 }, "topics":[{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }' ) curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index f3aefa6b9..a7de4ef8f 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -15,7 +15,7 @@ elastic_auth_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls - - mode: 600 + - mode: 640 - reload_pillar: True - contents: | elasticsearch: diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7b38ed0bb..023d270f4 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -162,6 +162,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -274,7 +275,7 @@ elasticsearch: number_of_replicas: 0 auto_expand_replicas: 0-2 number_of_shards: 1 - refresh_interval: 30s + refresh_interval: 1s sort: field: '@timestamp' order: desc @@ -316,6 +317,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -427,6 +429,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -534,6 +537,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -697,6 +701,7 @@ elasticsearch: - client-mappings - device-mappings - network-mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -768,6 +773,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -878,6 +884,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -998,6 +1005,7 @@ elasticsearch: index_template: composed_of: - so-data-streams-mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - so-logs-mappings @@ -2832,6 +2840,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3062,6 +3071,7 @@ elasticsearch: - event-mappings - logs-system.syslog@package - logs-system.syslog@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - so-system-mappings @@ -3421,6 +3431,7 @@ elasticsearch: - dtc-http-mappings - log-mappings - logstash-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3505,6 +3516,7 @@ elasticsearch: composed_of: - metrics-endpoint.metadata@package - metrics-endpoint.metadata@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3551,6 +3563,7 @@ elasticsearch: composed_of: - metrics-endpoint.metrics@package - metrics-endpoint.metrics@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3597,6 +3610,7 @@ elasticsearch: composed_of: - metrics-endpoint.policy@package - metrics-endpoint.policy@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3645,6 +3659,7 @@ elasticsearch: - metrics-fleet_server.agent_status@package - metrics-fleet_server.agent_status@custom - ecs@mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3668,6 +3683,7 @@ elasticsearch: - metrics-fleet_server.agent_versions@package - metrics-fleet_server.agent_versions@custom - ecs@mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3715,6 +3731,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3827,6 +3844,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3939,6 +3957,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4051,6 +4070,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4163,6 +4183,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4276,6 +4297,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index af162d9e9..b619854ed 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -204,12 +204,17 @@ so-elasticsearch-roles-load: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja -{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +{% if grains.role in ['so-managersearch', 'so-manager'] %} +{% set ap = "absent" %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} {% if ELASTICSEARCHMERGED.index_clean %} {% set ap = "present" %} {% else %} {% set ap = "absent" %} {% endif %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} so-elasticsearch-indices-delete: cron.{{ap}}: - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index e11a0be72..993c08373 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -8,7 +8,7 @@ "processors": [ { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, - { "split": { "if": "ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } }, + { "split": { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } }, { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, { "set": { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } }, { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, diff --git a/salt/elasticsearch/templates/component/ecs/log.json b/salt/elasticsearch/templates/component/ecs/log.json index e79661b5e..4f51be287 100644 --- a/salt/elasticsearch/templates/component/ecs/log.json +++ b/salt/elasticsearch/templates/component/ecs/log.json @@ -29,7 +29,7 @@ "file": { "properties": { "line": { - "type": "integer" + "type": "long" }, "name": { "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/ecs/metadata.json b/salt/elasticsearch/templates/component/ecs/metadata.json new file mode 100644 index 000000000..55da6f07c --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/metadata.json @@ -0,0 +1,26 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [], + "properties": { + "metadata": { + "properties": { + "kafka": { + "properties": { + "timestamp": { + "type": "date" + } + } + } + } + } + } + } + }, + "_meta": { + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html", + "ecs_version": "1.12.2" + } + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json index 5df7e7fe9..183031d4e 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json @@ -5,6 +5,7 @@ "managed_by": "security_onion", "managed": true }, + "date_detection": false, "dynamic_templates": [ { "strings_as_keyword": { @@ -16,7 +17,19 @@ } } ], - "date_detection": false + "properties": { + "metadata": { + "properties": { + "kafka": { + "properties": { + "timestamp": { + "type": "date" + } + } + } + } + } + } } }, "_meta": { diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json index 3777e670c..d6f516272 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json @@ -1,37 +1,59 @@ { - "template": { - "mappings": { - "properties": { - "host": { - "properties":{ - "ip": { - "type": "ip" - } - } - }, - "related": { - "properties":{ - "ip": { - "type": "ip" - } + "template": { + "mappings": { + "properties": { + "host": { + "properties": { + "ip": { + "type": "ip" } - }, - "destination": { - "properties":{ - "ip": { - "type": "ip" - } + } + }, + "related": { + "properties": { + "ip": { + "type": "ip" } - }, - "source": { - "properties":{ - "ip": { - "type": "ip" + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "metadata": { + "properties": { + "input": { + "properties": { + "beats": { + "properties": { + "host": { + "properties": { + "ip": { + "type": "ip" + } + } + } + } + } } } } } } } + }, + "_meta": { + "managed_by": "security_onion", + "managed": true } - \ No newline at end of file +} \ No newline at end of file diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index b9bfdbf63..e92a75bcb 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -11,6 +11,7 @@ firewall: endgame: [] eval: [] external_suricata: [] + external_kafka: [] fleet: [] heavynode: [] idh: [] @@ -103,6 +104,10 @@ firewall: tcp: - 9092 udp: [] + kafka_external_access: + tcp: + - 29092 + udp: [] kibana: tcp: - 5601 @@ -473,6 +478,8 @@ firewall: external_suricata: portgroups: - external_suricata + external_kafka: + portgroups: [] desktop: portgroups: - docker_registry @@ -668,6 +675,8 @@ firewall: external_suricata: portgroups: - external_suricata + external_kafka: + portgroups: [] desktop: portgroups: - docker_registry @@ -867,6 +876,8 @@ firewall: external_suricata: portgroups: - external_suricata + external_kafka: + portgroups: [] strelka_frontend: portgroups: - strelka_frontend @@ -1337,6 +1348,8 @@ firewall: endgame: portgroups: - endgame + external_kafka: + portgroups: [] receiver: portgroups: [] customhostgroup0: diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index fe04d7ad3..4347d2b31 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -21,25 +21,38 @@ {# Only add Kafka firewall items when Kafka enabled #} {% set role = GLOBALS.role.split('-')[1] %} -{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} -{% endif %} +{% if GLOBALS.pipeline == 'KAFKA' %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} +{% set kafka_node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %} -{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} -{% endif %} +{% if role in ['manager', 'managersearch', 'standalone'] %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} +{% endif %} -{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %} -{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %} -{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %} +{% if role == 'receiver' %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} +{% endif %} + +{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] %} +{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %} +{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %} +{% endif %} +{% endfor %} +{% endif %} + +{% if KAFKA_EXTERNAL_ACCESS %} +{# Kafka external access only applies for Kafka nodes with the broker role. #} +{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] and 'broker' in kafka_node_type %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.external_kafka.portgroups.append('kafka_external_access') %} {% endif %} -{% endfor %} +{% endif %} + {% endif %} -{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %} +{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %} \ No newline at end of file diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 222bcc8a2..6f0208b39 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -33,6 +33,7 @@ firewall: endgame: *hostgroupsettingsadv eval: *hostgroupsettings external_suricata: *hostgroupsettings + external_kafka: *hostgroupsettings fleet: *hostgroupsettings heavynode: *hostgroupsettings idh: *hostgroupsettings @@ -130,6 +131,9 @@ firewall: kafka_data: tcp: *tcpsettings udp: *udpsettings + kafka_external_access: + tcp: *tcpsettings + udp: *udpsettings kibana: tcp: *tcpsettings udp: *udpsettings diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls index 6d4b1036e..a44b02807 100644 --- a/salt/idstools/config.sls +++ b/salt/idstools/config.sls @@ -24,13 +24,23 @@ idstools_sbin: - group: 939 - file_mode: 755 -idstools_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://idstools/tools/sbin_jinja +# If this is used, exclude so-rule-update +#idstools_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://idstools/tools/sbin_jinja +# - user: 934 +# - group: 939 +# - file_mode: 755 +# - template: jinja + +idstools_so-rule-update: + file.managed: + - name: /usr/sbin/so-rule-update + - source: salt://idstools/tools/sbin_jinja/so-rule-update - user: 934 - group: 939 - - file_mode: 755 + - mode: 755 - template: jinja suricatacustomdirsfile: diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index 5e4c4c066..365b38772 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -55,6 +55,7 @@ so-idstools: {% endif %} - watch: - file: idstoolsetcsync + - file: idstools_so-rule-update delete_so-idstools_so-status.disabled: file.uncomment: @@ -76,6 +77,7 @@ run_so-rule-update: - require: - docker_container: so-idstools - onchanges: + - file: idstools_so-rule-update - file: idstoolsetcsync - file: synclocalnidsrules - order: last diff --git a/salt/influxdb/templates/alarm_nsm_disk.json b/salt/influxdb/templates/alarm_nsm_disk.json index 691f8c0e8..a3b10f7f3 100644 --- a/salt/influxdb/templates/alarm_nsm_disk.json +++ b/salt/influxdb/templates/alarm_nsm_disk.json @@ -5,10 +5,10 @@ "name": "alarm-nsm-disk" }, "spec": { - "description": "Percent used space on the root partition of at least one node has exceeded the alarm threshold.", + "description": "Percent used space on the nsm partition of at least one node has exceeded the alarm threshold.", "every": "1m0s", "name": "NSM Disk High Usage", - "query": "from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> aggregateWindow(every: 1m, fn: max, createEmpty: false)\n |> yield(name: \"max\")", + "query": "from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> aggregateWindow(every: 1m, fn: max, createEmpty: false)\n |> yield(name: \"max\")", "status": "active", "statusMessageTemplate": "Check: ${ r._check_name } is: ${ r._level }", "thresholds": [ diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index 1e43a3ec7..b8e299838 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -8,6 +8,7 @@ {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} {% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %} {% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} {# Create list of KRaft controllers #} {% set controllers = [] %} @@ -15,7 +16,7 @@ {# Check for Kafka nodes with controller in process_x_roles #} {% for node in KAFKA_NODES_PILLAR %} {% if 'controller' in KAFKA_NODES_PILLAR[node].role %} -{% do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ node ~ ":9093") %} +{% do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ KAFKA_NODES_PILLAR[node].ip ~ ":9093") %} {% endif %} {% endfor %} @@ -28,7 +29,15 @@ {# Generate server.properties for 'broker' , 'controller', 'broker,controller' node types anything above this line is a configuration needed for ALL Kafka nodes #} {% if node_type == 'broker' %} -{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% if KAFKA_EXTERNAL_ACCESS %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %} +{% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %} +{% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %} +{% else %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% endif %} {% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} {% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} {% do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} @@ -42,6 +51,7 @@ {% endif %} {% if node_type == 'controller' %} +{% do KAFKAMERGED.config.controller.update({'advertised_x_listeners': 'CONTROLLER://' + GLOBALS.node_ip + ':9093'}) %} {% do KAFKAMERGED.config.controller.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} {% do KAFKAMERGED.config.controller.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} {% do KAFKAMERGED.config.controller.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} @@ -50,7 +60,15 @@ {# Kafka nodes of this type are not recommended for use outside of development / testing. #} {% if node_type == 'broker,controller' %} -{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% if KAFKA_EXTERNAL_ACCESS %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %} +{% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %} +{% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %} +{% else %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' }) %} +{% endif %} {% do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %} {% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} {% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index e9222388b..1bedf96a0 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -6,6 +6,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} +{% set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users', default=None) %} kafka_group: group.present: @@ -69,6 +71,29 @@ kafka_kraft_{{sc}}_properties: - show_changes: False {% endfor %} +{% if KAFKA_EXTERNAL_ACCESS and KAFKA_EXTERNAL_USERS != None %} +kafka_server_jaas_properties: + file.managed: + - source: salt://kafka/etc/jaas.conf.jinja + - name: /opt/so/conf/kafka/kafka_server_jaas.conf + - template: jinja + - user: 960 + - group: 960 + - show_changes: False +{% else %} +remove_kafka_server_jaas_properties: + file.absent: + - name: /opt/so/conf/kafka/kafka_server_jaas.conf +{% endif %} + +kafka_log4j_properties: + file.managed: + - source: salt://kafka/etc/log4j.properties + - name: /opt/so/conf/kafka/log4j.properties + - user: 960 + - group: 960 + - show_changes: False + reset_quorum_on_changes: cmd.run: - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state @@ -81,4 +106,4 @@ reset_quorum_on_changes: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} +{% endif %} \ No newline at end of file diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index 21d6956ba..6b97ea84d 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -21,7 +21,7 @@ kafka: log_x_segment_x_bytes: 1073741824 node_x_id: num_x_io_x_threads: 8 - num_x_network_x_threads: 3 + num_x_network_x_threads: 5 num_x_partitions: 3 num_x_recovery_x_threads_x_per_x_data_x_dir: 1 offsets_x_topic_x_replication_x_factor: 1 @@ -46,6 +46,7 @@ kafka: ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: controller: + advertsied_x_listeners: controller_x_listener_x_names: CONTROLLER controller_x_quorum_x_voters: listeners: CONTROLLER://0.0.0.0:9093 @@ -61,4 +62,10 @@ kafka: ssl_x_keystore_x_password: ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks ssl_x_truststore_x_type: JKS - ssl_x_truststore_x_password: \ No newline at end of file + ssl_x_truststore_x_password: + external_access: + enabled: False + listeners: EXTERNAL_ACCESS://0.0.0.0:29092 + listener_x_security_x_protocol_x_map: EXTERNAL_ACCESS:SASL_SSL + sasl_x_enabled_x_mechanisms: PLAIN + sasl_x_mechanism_x_inter_x_broker_x_protocol: SSL \ No newline at end of file diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index 362f7fde3..8448bd5aa 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -14,6 +14,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} {% set KAFKANODES = salt['pillar.get']('kafka:nodes') %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} {% if 'gmd' in salt['pillar.get']('features', []) %} include: @@ -34,7 +35,7 @@ so-kafka: - user: kafka - environment: KAFKA_HEAP_OPTS: -Xmx2G -Xms1G - KAFKA_OPTS: -javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml + KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}" - extra_hosts: {% for node in KAFKANODES %} - {{ node }}:{{ KAFKANODES[node].ip }} @@ -54,11 +55,18 @@ so-kafka: - /nsm/kafka/data/:/nsm/kafka/data/:rw - /opt/so/log/kafka:/opt/kafka/logs/:rw - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro - - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties + - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties:ro + - /opt/so/conf/kafka/log4j.properties:/opt/kafka/config/log4j.properties:ro + {% if KAFKA_EXTERNAL_ACCESS %} + - /opt/so/conf/kafka/kafka_server_jaas.conf:/opt/kafka/config/kafka_server_jaas.conf:ro + {% endif %} - watch: {% for sc in ['server', 'client'] %} - file: kafka_kraft_{{sc}}_properties {% endfor %} + {% if KAFKA_EXTERNAL_ACCESS %} + - file: kafka_server_jaas_properties + {% endif %} - file: kafkacertz - require: - file: kafkacertz @@ -87,4 +95,4 @@ include: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} +{% endif %} \ No newline at end of file diff --git a/salt/kafka/etc/jaas.conf.jinja b/salt/kafka/etc/jaas.conf.jinja new file mode 100644 index 000000000..e12367cd9 --- /dev/null +++ b/salt/kafka/etc/jaas.conf.jinja @@ -0,0 +1,16 @@ +{% set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users') -%} + +{%- set valid_users = [] -%} + +{%- for item, user in KAFKA_EXTERNAL_USERS.items() -%} +{% if 'password' in user and user.password is not none and user.password.strip() != "" -%} +{% do valid_users.append('user_' ~ user.username ~ '="' ~ user.password ~ '"') -%} +{% endif -%} +{%- endfor -%} + +KafkaServer { + org.apache.kafka.common.security.plain.PlainLoginModule required + {% for user_entry in valid_users -%} + {{ user_entry }}{{ ";" if loop.last }} + {% endfor %} +}; \ No newline at end of file diff --git a/salt/kafka/etc/log4j.properties b/salt/kafka/etc/log4j.properties new file mode 100644 index 000000000..76194c1ee --- /dev/null +++ b/salt/kafka/etc/log4j.properties @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Unspecified loggers and loggers with additivity=true output to server.log and stdout +# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise +log4j.rootLogger=INFO, stdout, kafkaAppender + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.MaxFileSize=10MB +log4j.appender.kafkaAppender.MaxBackupIndex=10 +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.MaxFileSize=10MB +log4j.appender.stateChangeAppender.MaxBackupIndex=10 +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.MaxFileSize=10MB +log4j.appender.requestAppender.MaxBackupIndex=10 +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.cleanerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.MaxFileSize=10MB +log4j.appender.cleanerAppender.MaxBackupIndex=10 +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.MaxFileSize=10MB +log4j.appender.controllerAppender.MaxBackupIndex=10 +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.authorizerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log +log4j.appender.authorizerAppender.MaxFileSize=10MB +log4j.appender.authorizerAppender.MaxBackupIndex=10 +log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Change the line below to adjust ZK client logging +log4j.logger.org.apache.zookeeper=INFO + +# Change the two lines below to adjust the general broker logging level (output to server.log and stdout) +log4j.logger.kafka=INFO +log4j.logger.org.apache.kafka=INFO + +# Change to DEBUG or TRACE to enable request logging +log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.additivity.kafka.request.logger=false + +# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output +# related to the handling of requests +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +# Change the line below to adjust KRaft mode controller logging +log4j.logger.org.apache.kafka.controller=INFO, controllerAppender +log4j.additivity.org.apache.kafka.controller=false + +# Change the line below to adjust ZK mode controller logging +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=INFO, stateChangeAppender +log4j.additivity.state.change.logger=false + +# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses +log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender +log4j.additivity.kafka.authorizer.logger=false \ No newline at end of file diff --git a/salt/kafka/nodes.sls b/salt/kafka/nodes.sls index cae2a1d0f..90cc931bb 100644 --- a/salt/kafka/nodes.sls +++ b/salt/kafka/nodes.sls @@ -10,9 +10,9 @@ write_kafka_pillar_yaml: file.managed: - name: /opt/so/saltstack/local/pillar/kafka/nodes.sls - - mode: 644 + - mode: 640 - user: socore - source: salt://kafka/files/managed_node_pillar.jinja - template: jinja - context: - COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }} \ No newline at end of file + COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }} diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 8087f9bdf..cb093600f 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -34,10 +34,6 @@ kafka: sensitive: True helpLink: kafka.html broker: - advertised_x_listeners: - description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication. - title: advertised.listeners - helpLink: kafka.html auto_x_create_x_topics_x_enable: description: Enable the auto creation of topics. title: auto.create.topics.enable @@ -226,4 +222,52 @@ kafka: description: The role performed by controller node. title: process.roles readonly: True - helpLink: kafka.html \ No newline at end of file + helpLink: kafka.html + external_access: + enabled: + description: Enables or disables access to Kafka topics using user/password authentication. Used for producing / consuming messages via an external client. + forcedType: bool + helpLink: kafka.html + listeners: + description: Set of URIs that is listened on and the listener names in a comma-seperated list. + title: listeners + readonly: True + advanced: True + helpLink: kafka.html + listener_x_security_x_protocol_x_map: + description: External listener name and mapped security protocol. + title: listener.security.protocol.map + readonly: True + advanced: True + helpLink: kafka.html + sasl_x_enabled_x_mechanisms: + description: SASL/PLAIN is a simple username/password authentication mechanism, used with TLS to implement secure authentication. + title: sasl.enabled.mechanisms + readonly: True + advanced: True + helpLink: kafka.html + sasl_x_mechanism_x_inter_x_broker_x_protocol: + description: SASL mechanism used for inter-broker communication + title: sasl.mechanism.inter.broker.protocol + readonly: True + advanced: True + helpLink: kafka.html + remote_users: + user01: &remote_user + username: + description: Username to be used for custom account + forcedType: string + global: True + password: + description: Password to be used for custom account + forcedType: string + global: True + sensitive: True + user02: *remote_user + user03: *remote_user + user04: *remote_user + user05: *remote_user + user06: *remote_user + user07: *remote_user + user08: *remote_user + user09: *remote_user \ No newline at end of file diff --git a/salt/kibana/secrets.sls b/salt/kibana/secrets.sls index f97aa4d59..048cea4d4 100644 --- a/salt/kibana/secrets.sls +++ b/salt/kibana/secrets.sls @@ -22,7 +22,7 @@ kibana_pillar_directory: kibana_secrets_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/kibana/secrets.sls - - mode: 600 + - mode: 640 - reload_pillar: True - contents: | kibana: diff --git a/salt/kratos/config.sls b/salt/kratos/config.sls index 0be43b460..b9f5142f1 100644 --- a/salt/kratos/config.sls +++ b/salt/kratos/config.sls @@ -13,13 +13,21 @@ kratosgroup: - name: kratos - gid: 928 +kratoshome: + file.directory: + - name: /opt/so/conf/kratos + - user: 928 + - group: 928 + - mode: 700 + - makedirs: True + # Add Kratos user kratos: user.present: - uid: 928 - gid: 928 - home: /opt/so/conf/kratos - + kratosdir: file.directory: - name: /nsm/kratos diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 8a59c83b7..5a1727e9b 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -22,7 +22,15 @@ logstashgroup: - name: logstash - gid: 931 -# Add the logstash user for the jog4j settings +logstashhome: + file.directory: + - name: /opt/so/conf/logstash + - user: 931 + - group: 931 + - mode: 700 + - makedirs: True + +# Add the logstash user for the log4j settings logstash: user.present: - uid: 931 diff --git a/salt/logstash/etc/log4j2.properties b/salt/logstash/etc/log4j2.properties index 739756061..750a6e316 100644 --- a/salt/logstash/etc/log4j2.properties +++ b/salt/logstash/etc/log4j2.properties @@ -23,6 +23,8 @@ appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy appender.rolling.policies.time.interval = 1 appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 1GB appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.action.type = Delete appender.rolling.strategy.action.basepath = /var/log/logstash diff --git a/salt/manager/elasticsearch.sls b/salt/manager/elasticsearch.sls index df93217b8..ab9dbb287 100644 --- a/salt/manager/elasticsearch.sls +++ b/salt/manager/elasticsearch.sls @@ -3,5 +3,5 @@ elastic_curl_config_distributed: - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config - source: salt://elasticsearch/files/curl.config.template - template: jinja - - mode: 600 + - mode: 640 - show_changes: False diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 5eadead92..07a1b8816 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -127,15 +127,28 @@ so_fleetagent_status: - month: '*' - dayweek: '*' -socore_own_saltstack: +socore_own_saltstack_default: file.directory: - - name: /opt/so/saltstack + - name: /opt/so/saltstack/default - user: socore - group: socore - recurse: - user - group +socore_own_saltstack_local: + file.directory: + - name: /opt/so/saltstack/local + - user: socore + - group: socore + - dir_mode: 750 + - file_mode: 640 + - replace: False + - recurse: + - user + - group + - mode + rules_dir: file.directory: - name: /nsm/rules/yara diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ebbfa8fff..98add9a96 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -126,7 +126,7 @@ function testMinion() { } function restartMinion() { - salt "$MINION_ID" system.reboot + salt "$MINION_ID" system.reboot --async result=$? exit $result diff --git a/salt/manager/tools/sbin/so-user b/salt/manager/tools/sbin/so-user index e6ac9eb1f..92b3ba385 100755 --- a/salt/manager/tools/sbin/so-user +++ b/salt/manager/tools/sbin/so-user @@ -356,7 +356,7 @@ function syncElastic() { [[ $? != 0 ]] && fail "Unable to read credential hashes from database" user_data_formatted=$(echo "${userData}" | jq -r '.user + ":" + .data.hashed_password') - if lookup_salt_value "features" "" "pillar" | grep -x odc; then + if lookup_salt_value "features" "" "pillar" | grep -qx odc; then # generate random placeholder salt/hash for users without passwords random_crypt=$(get_random_value 53) user_data_formatted=$(echo "${user_data_formatted}" | sed -r "s/^(.+:)\$/\\1\$2a\$12${random_crypt}/") diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c0b7f6e1c..7022bf5c0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -243,6 +243,13 @@ check_pillar_items() { fi } +check_saltmaster_status() { + set +e + echo "Waiting on the Salt Master service to be ready." + check_salt_master_status || fail "Can't access salt master or it is not ready. Check $SOUP_LOG for details." + set -e +} + check_sudoers() { if grep -q "so-setup" /etc/sudoers; then echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." @@ -409,6 +416,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.120 ]] && up_to_2.4.130 [[ "$INSTALLEDVERSION" == 2.4.130 ]] && up_to_2.4.140 [[ "$INSTALLEDVERSION" == 2.4.140 ]] && up_to_2.4.141 + [[ "$INSTALLEDVERSION" == 2.4.141 ]] && up_to_2.4.150 true } @@ -435,6 +443,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.120 ]] && post_to_2.4.130 [[ "$POSTVERSION" == 2.4.130 ]] && post_to_2.4.140 [[ "$POSTVERSION" == 2.4.140 ]] && post_to_2.4.141 + [[ "$POSTVERSION" == 2.4.141 ]] && post_to_2.4.150 true } @@ -551,9 +560,6 @@ post_to_2.4.130() { echo "Updating Kibana default space" /usr/sbin/so-kibana-space-defaults - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers - POSTVERSION=2.4.130 } @@ -567,6 +573,12 @@ post_to_2.4.141() { POSTVERSION=2.4.141 } +post_to_2.4.150() { + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers + POSTVERSION=2.4.150 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -739,8 +751,6 @@ up_to_2.4.90() { so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" - echo "If the Detection index exists, update the refresh_interval" - so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.90 } @@ -799,6 +809,13 @@ up_to_2.4.141() { INSTALLEDVERSION=2.4.141 } +up_to_2.4.150() { + echo "If the Detection indices exists, update the refresh_interval" + so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' + + INSTALLEDVERSION=2.4.150 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls @@ -1411,10 +1428,7 @@ main() { systemctl_func "start" "salt-master" # Testing that salt-master is up by checking that is it connected to itself - set +e - echo "Waiting on the Salt Master service to be ready." - check_salt_master_status || fail "Can't access salt master or it is not ready. Check $SOUP_LOG for details." - set -e + check_saltmaster_status # update the salt-minion configs here and start the minion # since highstate are disabled above, minion start should not trigger a highstate @@ -1441,10 +1455,7 @@ main() { systemctl_func "start" "salt-master" - set +e - echo "Waiting on the Salt Master service to be ready." - check_salt_master_status || fail "Can't access salt master or it is not ready. Check $SOUP_LOG for details." - set -e + check_saltmaster_status echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG" @@ -1456,6 +1467,7 @@ main() { update_salt_mine highstate + check_saltmaster_status postupgrade_changes [[ $is_airgap -eq 0 ]] && unmount_update diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index e2bcef863..9c34724f6 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -63,7 +63,7 @@ managerssl_crt: - signing_policy: managerssl - private_key: /etc/pki/managerssl.key - CN: {{ GLOBALS.hostname }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - subjectAltName: "DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}, DNS:{{ GLOBALS.url_base }}" - days_remaining: 0 - days_valid: 820 - backup: True @@ -121,7 +121,7 @@ so-nginx: - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - - /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/:/opt/socore/html/packages + - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %} - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro diff --git a/salt/salt/master/mine_update_highstate.sls b/salt/salt/master/mine_update_highstate.sls new file mode 100644 index 000000000..c91be20e9 --- /dev/null +++ b/salt/salt/master/mine_update_highstate.sls @@ -0,0 +1,29 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# This state should only be run on managers and should never be run manually + +{% set MINION_ID = grains.id %} + +# Run mine.update on all minions +salt.master.mine_update_highstate.update_mine_all_minions: + salt.function: + - name: mine.update + - tgt: '*' + - batch: 50 + - retry: + attempts: 3 + interval: 1 + +# Run highstate on the original minion +# we can use concurrent on this highstate because no other highstate would be running when this is called +# this state will run onlyif there is not an instance of it already running +salt.master.mine_update_highstate.run_highstate_on_{{ MINION_ID }}: + salt.state: + - tgt: {{ MINION_ID }} + - highstate: True + - concurrent: True + - onlyif: + - 'ps -ef | grep -v grep | grep "/usr/bin/salt-minion.*ProcessPayload.*jid=.*Minion._thread_return" | wc -l | grep -q "^0$"' diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 68ae04804..7ec839950 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -3,4 +3,3 @@ salt: minion: version: '3006.9' check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default - service_start_delay: 30 # in seconds. diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index ee7c1f14c..9f2a6f3fa 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -5,10 +5,10 @@ {% from 'salt/map.jinja' import SALTPACKAGES %} {% from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %} {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} -{% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} include: - salt.python_modules + - salt.patch.x509_v2 - salt - systemd.reload - repo.client @@ -89,8 +89,6 @@ salt_minion_service_unit_file: - name: {{ SYSTEMD_UNIT_FILE }} - source: salt://salt/service/salt-minion.service.jinja - template: jinja - - defaults: - service_start_delay: {{ service_start_delay }} - onchanges_in: - module: systemd_reload diff --git a/salt/salt/patch/x509_v2/init.sls b/salt/salt/patch/x509_v2/init.sls new file mode 100644 index 000000000..2483339a2 --- /dev/null +++ b/salt/salt/patch/x509_v2/init.sls @@ -0,0 +1,6 @@ +patch_x509_v2_state_module: + file.replace: + - name: /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py + - pattern: 'res = __salt__\["state.single"\]\("file.managed", name, test=test, \*\*kwargs\)' + - repl: 'res = __salt__["state.single"]("file.managed", name, test=test, concurrent=True, **kwargs)' + - backup: .bak diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja index 27c7a15b6..2763b30f5 100644 --- a/salt/salt/service/salt-minion.service.jinja +++ b/salt/salt/service/salt-minion.service.jinja @@ -8,8 +8,9 @@ KillMode=process Type=notify NotifyAccess=all LimitNOFILE=8192 +ExecStartPre=/bin/bash -c 'until /sbin/ip -4 addr show dev {{ salt["pillar.get"]("host:mainint") }} | grep -q "inet "; do sleep 1; done' ExecStart=/usr/bin/salt-minion -ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} +TimeoutStartSec=120 [Install] WantedBy=multi-user.target diff --git a/salt/sensoroni/files/analyzers/echotrail/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/echotrail/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index dac873718..000000000 Binary files a/salt/sensoroni/files/analyzers/echotrail/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/echotrail/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/echotrail/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/echotrail/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/echotrail/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/echotrail/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index b1cd02e9d..000000000 Binary files a/salt/sensoroni/files/analyzers/echotrail/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/echotrail/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/echotrail/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/echotrail/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/elasticsearch/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/elasticsearch/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index dac873718..000000000 Binary files a/salt/sensoroni/files/analyzers/elasticsearch/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/elasticsearch/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/elasticsearch/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/elasticsearch/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/elasticsearch/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/elasticsearch/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index b1cd02e9d..000000000 Binary files a/salt/sensoroni/files/analyzers/elasticsearch/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/elasticsearch/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/elasticsearch/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/elasticsearch/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/emailrep/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/emailrep/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/emailrep/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/emailrep/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/emailrep/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/emailrep/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/greynoise/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/greynoise/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/greynoise/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/greynoise/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/greynoise/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/greynoise/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/localfile/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/localfile/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/localfile/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/localfile/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/localfile/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/localfile/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index dac873718..000000000 Binary files a/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index b1cd02e9d..000000000 Binary files a/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/malwarebazaar/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/requirements.txt b/salt/sensoroni/files/analyzers/malwarehashregistry/requirements.txt index dfaf321f1..a5a02f4fa 100644 --- a/salt/sensoroni/files/analyzers/malwarehashregistry/requirements.txt +++ b/salt/sensoroni/files/analyzers/malwarehashregistry/requirements.txt @@ -1,2 +1,2 @@ requests>=2.31.0 -python-whois>=0.7.3 +python-whois>=0.9.5 diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/future-0.18.3.tar.gz b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/future-0.18.3.tar.gz deleted file mode 100644 index 9ca264a4f..000000000 Binary files a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/future-0.18.3.tar.gz and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python-whois-0.8.0.tar.gz b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python-whois-0.8.0.tar.gz deleted file mode 100644 index ffa2e5ded..000000000 Binary files a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python-whois-0.8.0.tar.gz and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python_dateutil-2.9.0.post0-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python_dateutil-2.9.0.post0-py2.py3-none-any.whl new file mode 100644 index 000000000..b9a14e1bf Binary files /dev/null and b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python_dateutil-2.9.0.post0-py2.py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python_whois-0.9.5-py3-none-any.whl b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python_whois-0.9.5-py3-none-any.whl new file mode 100644 index 000000000..d92bc8729 Binary files /dev/null and b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/python_whois-0.9.5-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/six-1.17.0-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/six-1.17.0-py2.py3-none-any.whl new file mode 100644 index 000000000..c506fd05b Binary files /dev/null and b/salt/sensoroni/files/analyzers/malwarehashregistry/source-packages/six-1.17.0-py2.py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/otx/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/otx/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/otx/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/otx/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/otx/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/otx/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/pulsedive/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/pulsedive/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/pulsedive/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/pulsedive/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/pulsedive/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/pulsedive/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2021.10.8-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2021.10.8-py2.py3-none-any.whl deleted file mode 100644 index fbcb86b5f..000000000 Binary files a/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2021.10.8-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2023.5.7-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2023.5.7-py3-none-any.whl new file mode 100644 index 000000000..c983e799c Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2023.5.7-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-2.0.12-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-2.0.12-py3-none-any.whl deleted file mode 100644 index 17a2dfbeb..000000000 Binary files a/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-2.0.12-py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.3-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.3-py3-none-any.whl deleted file mode 100644 index 060541bc9..000000000 Binary files a/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.3-py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.4-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.4-py3-none-any.whl new file mode 100644 index 000000000..7343c6845 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.4-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.27.1-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.27.1-py2.py3-none-any.whl deleted file mode 100644 index 807fc6110..000000000 Binary files a/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.27.1-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.31.0-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.31.0-py3-none-any.whl new file mode 100644 index 000000000..bfd5d2ea9 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.31.0-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-1.26.9-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-1.26.9-py2.py3-none-any.whl deleted file mode 100644 index 5019453dd..000000000 Binary files a/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-1.26.9-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-2.0.3-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-2.0.3-py3-none-any.whl new file mode 100644 index 000000000..5e0b52889 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-2.0.3-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/threatfox/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/threatfox/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index dac873718..000000000 Binary files a/salt/sensoroni/files/analyzers/threatfox/source-packages/PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/threatfox/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/threatfox/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/threatfox/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/threatfox/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/threatfox/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index b1cd02e9d..000000000 Binary files a/salt/sensoroni/files/analyzers/threatfox/source-packages/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/threatfox/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/threatfox/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/threatfox/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/urlhaus/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/urlhaus/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/urlhaus/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/urlhaus/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/urlhaus/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/urlhaus/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/urlscan/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/urlscan/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/urlscan/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/urlscan/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/urlscan/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/urlscan/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..e7e59e816 Binary files /dev/null and b/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/virustotal/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/virustotal/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/virustotal/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/virustotal/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/whoislookup/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/whoislookup/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index f125c08aa..000000000 Binary files a/salt/sensoroni/files/analyzers/whoislookup/source-packages/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/whoislookup/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/sensoroni/files/analyzers/whoislookup/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..666649ed2 Binary files /dev/null and b/salt/sensoroni/files/analyzers/whoislookup/source-packages/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/whoislookup/source-packages/setuptools-80.1.0-py3-none-any.whl b/salt/sensoroni/files/analyzers/whoislookup/source-packages/setuptools-80.1.0-py3-none-any.whl new file mode 100644 index 000000000..98164f3da Binary files /dev/null and b/salt/sensoroni/files/analyzers/whoislookup/source-packages/setuptools-80.1.0-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/whoislookup/whoislookup_test.py b/salt/sensoroni/files/analyzers/whoislookup/whoislookup_test.py index 9aa596095..61cd49a68 100755 --- a/salt/sensoroni/files/analyzers/whoislookup/whoislookup_test.py +++ b/salt/sensoroni/files/analyzers/whoislookup/whoislookup_test.py @@ -36,7 +36,7 @@ class TestWhoisLookupMethods(unittest.TestCase): def test_sendReqNotFound(self): mock = MagicMock() - mock.side_effect = whoisit.errors.ResourceDoesNotExist() + mock.side_effect = whoisit.errors.ResourceDoesNotExist("foo") with patch('whoisit.domain', new=mock): response = whoislookup.sendReq("abcd1234.com") mock.assert_called_once_with("abcd1234.com", raw=True) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 5a8ec840c..242050f98 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -10,7 +10,7 @@ soc: icon: fa-crosshairs target: links: - - '/#/hunt?q="{value|escape}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{value|escape}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' - name: actionAddToCase description: actionAddToCaseHelp icon: fa-briefcase @@ -24,20 +24,20 @@ soc: icon: fa-magnifying-glass-arrow-right target: '' links: - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - - '/#/hunt?q="{:log.id.uid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - - '/#/hunt?q="{:network.community_id}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' + - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' + - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' + - '/#/hunt?q="{:log.id.uid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' + - '/#/hunt?q="{:network.community_id}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}' - name: actionPcap description: actionPcapHelp icon: fa-stream target: '' links: - - '/joblookup?esid={:soc_id}&time={:@timestamp}' - - '/joblookup?ncid={:network.community_id}&time={:@timestamp}' + - '/joblookup?esid={:soc_id}&time={:@timestamp}&gridId={gridId}' + - '/joblookup?ncid={:network.community_id}&time={:@timestamp}&gridId={gridId}' categories: - hunt - alerts @@ -71,30 +71,30 @@ soc: icon: fa-person-running target: '' links: - - '/#/hunt?q=(process.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path&gridId={gridId}' - name: actionProcessChildInfo description: actionProcessChildInfoHelp icon: fa-users-line target: '' links: - - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.parent.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.parent.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path&gridId={gridId}' - name: actionProcessAllInfo description: actionProcessAllInfoHelp icon: fa-users-between-lines target: '' links: - - '/#/hunt?q="{:process.entity_id}" | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' + - '/#/hunt?q="{:process.entity_id}" | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path&gridId={gridId}' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof target: '' links: - - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path&gridId={gridId}' - name: actionRelatedAlerts description: actionRelatedAlertsHelp icon: fa-bell links: - - '/#/alerts?q=rule.uuid: {:so_detection.publicId|escape} | groupby rule.name event.module* event.severity_label' + - '/#/alerts?q=rule.uuid: {:so_detection.publicId|escape} | groupby rule.name event.module* event.severity_label&gridId={gridId}' target: '' - name: actionAdd description: actionAddHelp @@ -116,14 +116,14 @@ soc: - soc_timestamp - event.dataset - http_request.headers.x-real-ip - - identity_id + - user.name - http_request.headers.user-agent - msg ':hydra:': - soc_timestamp - event.dataset - http_request.headers.x-real-ip - - identity_id + - user.name - http_request.headers.user-agent - msg '::conn': @@ -1605,7 +1605,7 @@ soc: showSubtitle: true - name: SOC - Auth description: Users authenticated to SOC grouped by IP address and identity - query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip identity_id' + query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip user.name' showSubtitle: true - name: SOC - App description: Logs generated by the Security Onion Console (SOC) server and modules @@ -1885,7 +1885,7 @@ soc: query: '* | groupby event.category | groupby -sankey event.category event.module | groupby event.module | groupby -sankey event.module event.dataset | groupby event.dataset | groupby observer.name | groupby host.name | groupby source.ip | groupby destination.ip | groupby destination.port' - name: SOC Logins description: SOC (Security Onion Console) logins - query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip identity_id | groupby identity_id | groupby http_request.headers.user-agent' + query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip user.name | groupby user.name | groupby http_request.headers.user-agent' - name: SOC Login Failures description: SOC (Security Onion Console) login failures query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip http_request.headers.user-agent | groupby http_request.headers.user-agent' diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b0ecfdbc1..91ab6e3c1 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -81,6 +81,30 @@ soc: label: Target - field: jscall label: JavaScript Call + - field: background + label: Background XHR Request + forcedType: bool + - field: method + label: XHR Method + options: + - DELETE + - GET + - PATCH + - POST + - PUT + - field: options + label: XHR Options (JSON) + multiline: True + forcedType: "{}" + - field: body + label: XHR Content + - field: encodeBody + label: Encode XHR Content Variable Data + forcedType: bool + - field: backgroundSuccessLink + label: XHR Success Link + - field: backgroundFailureLink + label: XHR Failure Link - field: category label: Category options: @@ -114,6 +138,43 @@ soc: title: Require TOTP description: Require all users to enable Time-based One Time Passwords (MFA) upon login to SOC. global: True + subgrids: + title: Subordinate Grids + description: | + Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must be granted most permissions in order to perform required functions. + + *Requires a valid Security Onion license key with subgrid allocations.* + global: True + syntax: json + forcedType: "[]{}" + uiElements: + - field: id + label: Unique Subgrid ID + regex: "^((?!_)).+$" + regexFailureMessage: Subgrid ID cannot start with an underscore + required: true + - field: managerUrl + label: Subgrid Manager URL + required: true + - field: clientId + label: Subgrid API Client ID + required: true + regex: "^socl_[a-z0-9_]+$" + regexFailureMessage: Client ID must be a valid socl_* API Client ID + - field: clientSecret + label: Subgrid API Client Secret + required: true + - field: tlsSkipVerify + label: Skip Subgrid TLS Certification Validation + forcedType: bool + default: false + - field: caCertificate + label: Subgrid CA Certificate + multiline: True + - field: enabled + label: Subgrid Enabled + forcedType: bool + default: false modules: elastalertengine: aiRepoUrl: diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index a26905e1f..3a830c9b0 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -44,6 +44,10 @@ strelka_backend: - restart_policy: on-failure - watch: - file: strelkasensorcompiledrules + - file: backend_backend_config + - file: backend_logging_config + - file: backend_passwords + - file: backend_taste delete_so-strelka-backend_so-status.disabled: file.uncomment: diff --git a/salt/strelka/filestream/enabled.sls b/salt/strelka/filestream/enabled.sls index f04631eca..c90b1e83f 100644 --- a/salt/strelka/filestream/enabled.sls +++ b/salt/strelka/filestream/enabled.sls @@ -41,6 +41,8 @@ strelka_filestream: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - file: filestream_config delete_so-strelka-filestream_so-status.disabled: file.uncomment: diff --git a/salt/strelka/frontend/enabled.sls b/salt/strelka/frontend/enabled.sls index e4ecc7ca5..f95a31a7e 100644 --- a/salt/strelka/frontend/enabled.sls +++ b/salt/strelka/frontend/enabled.sls @@ -46,6 +46,8 @@ strelka_frontend: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - file: frontend_config delete_so-strelka-frontend_so-status.disabled: file.uncomment: diff --git a/salt/strelka/manager/enabled.sls b/salt/strelka/manager/enabled.sls index aec44b4b0..6158a5c28 100644 --- a/salt/strelka/manager/enabled.sls +++ b/salt/strelka/manager/enabled.sls @@ -40,6 +40,8 @@ strelka_manager: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - file: manager_config delete_so-strelka-manager_so-status.disabled: file.uncomment: diff --git a/salt/strelka/soc_strelka.yaml b/salt/strelka/soc_strelka.yaml index 1dc4fa455..1a5db261b 100644 --- a/salt/strelka/soc_strelka.yaml +++ b/salt/strelka/soc_strelka.yaml @@ -64,46 +64,62 @@ strelka: helpLink: strelka.html advanced: True scanners: - 'ScanBase64': &scannerOptions + 'ScanBase64PE': &scannerOptions description: Configuration options for this scanner. readonly: False global: False helpLink: strelka.html advanced: True - type: json - multiline: True + forcedType: "[]{}" + syntax: json 'ScanBatch': *scannerOptions + 'ScanBmpEof': *scannerOptions 'ScanBzip2': *scannerOptions + 'ScanDmg': *scannerOptions 'ScanDocx': *scannerOptions + 'ScanDonut': *scannerOptions 'ScanElf': *scannerOptions 'ScanEmail': *scannerOptions + 'ScanEncryptedDoc': *scannerOptions + 'ScanEncryptedZip': *scannerOptions 'ScanEntropy': *scannerOptions 'ScanExiftool': *scannerOptions + 'ScanFooter': *scannerOptions 'ScanGif': *scannerOptions 'ScanGzip': *scannerOptions 'ScanHash': *scannerOptions 'ScanHeader': *scannerOptions 'ScanHtml': *scannerOptions 'ScanIni': *scannerOptions + 'ScanIqy': *scannerOptions + 'ScanIso': *scannerOptions 'ScanJarManifest': *scannerOptions 'ScanJavascript': *scannerOptions 'ScanJpeg': *scannerOptions 'ScanJson': *scannerOptions 'ScanLibarchive': *scannerOptions + 'ScanLNK': *scannerOptions + 'ScanLsb': *scannerOptions 'ScanLzma': *scannerOptions 'ScanMacho': *scannerOptions + 'ScanManifest': *scannerOptions + 'ScanMsi': *scannerOptions 'ScanOcr': *scannerOptions 'ScanOle': *scannerOptions + 'ScanOnenote': *scannerOptions 'ScanPdf': *scannerOptions 'ScanPe': *scannerOptions 'ScanPgp': *scannerOptions 'ScanPhp': *scannerOptions 'ScanPkcs7': *scannerOptions 'ScanPlist': *scannerOptions + 'ScanPngEof': *scannerOptions + 'ScanQr': *scannerOptions 'ScanRar': *scannerOptions 'ScanRpm': *scannerOptions 'ScanRtf': *scannerOptions 'ScanRuby': *scannerOptions + 'ScanSevenZip': *scannerOptions 'ScanSwf': *scannerOptions 'ScanTar': *scannerOptions 'ScanTnef': *scannerOptions @@ -111,6 +127,8 @@ strelka: 'ScanUrl': *scannerOptions 'ScanVb': *scannerOptions 'ScanVba': *scannerOptions + 'ScanVhd': *scannerOptions + 'ScanVsto': *scannerOptions 'ScanX509': *scannerOptions 'ScanXml': *scannerOptions 'ScanYara': *scannerOptions diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index a35be55f5..171bd41f5 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -45,6 +45,24 @@ tgraf_sync_script_{{script}}: GLOBALS: {{ GLOBALS }} {% endfor %} +{% if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %} +tgraf_sync_script_esindexsize.sh: + file.managed: + - name: /opt/so/conf/telegraf/scripts/esindexsize.sh + - user: root + - group: 939 + - mode: 770 + - source: salt://telegraf/scripts/esindexsize.sh +{# Copy conf/elasticsearch/curl.config for telegraf to use with esindexsize.sh #} +tgraf_sync_escurl_conf: + file.managed: + - name: /opt/so/conf/telegraf/etc/escurl.config + - user: 939 + - group: 939 + - mode: 400 + - source: salt://elasticsearch/curl.config +{% endif %} + telegraf_sbin: file.recurse: - name: /usr/sbin diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index 8e3bc9fbe..451c78dda 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -20,7 +20,6 @@ so-telegraf: - user: 939 - group_add: 939,920 - environment: - - HOST_PROC=/host/proc - HOST_ETC=/host/etc - HOST_SYS=/host/sys - HOST_MOUNT_PREFIX=/host @@ -38,7 +37,7 @@ so-telegraf: - /opt/so/conf/telegraf/node_config.json:/etc/telegraf/node_config.json:ro - /var/run/utmp:/var/run/utmp:ro - /var/run/docker.sock:/var/run/docker.sock:ro - - /:/host/root:ro + - /:/host:ro - /sys:/host/sys:ro - /proc:/host/proc:ro - /nsm:/host/nsm:ro @@ -57,6 +56,9 @@ so-telegraf: - /opt/so/log/sostatus:/var/log/sostatus:ro - /opt/so/log/salt:/var/log/salt:ro - /opt/so/log/agents:/var/log/agents:ro + {% if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %} + - /opt/so/conf/telegraf/etc/escurl.config:/etc/telegraf/elasticsearch.config:ro + {% endif %} {% if DOCKER.containers['so-telegraf'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-telegraf'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 533627bbd..f5d331209 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -113,7 +113,13 @@ [[inputs.disk]] ## By default stats will be gathered for all mount points. ## Set mount_points will restrict the stats to only the specified mount points. - #mount_points = ["/", "/host/nsm"] + mount_points = ["/host", + "/host/nsm", + "/host/var", + "/host/var/log", + "/host/var/log/audit", + "/host/var/tmp" + ] ## Ignore mount points by filesystem type. #ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"] @@ -186,7 +192,7 @@ # # # Read stats from one or more Elasticsearch servers or clusters -{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-import'] %} +{%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = true @@ -317,3 +323,13 @@ # # Read metrics about network interface usage [[inputs.net]] + +# Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently +{%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %} +[[ inputs.exec ]] + commands = [ + "/scripts/esindexsize.sh" + ] + data_format = "influx" + interval = "1h" +{%- endif %} diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh index a390552fc..718f0e5ce 100644 --- a/salt/telegraf/scripts/agentstatus.sh +++ b/salt/telegraf/scripts/agentstatus.sh @@ -24,7 +24,7 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',') TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',') ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',') - ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}') + ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}' | tr -d ',') echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE" fi diff --git a/salt/telegraf/scripts/esindexsize.sh b/salt/telegraf/scripts/esindexsize.sh new file mode 100644 index 000000000..dbb50f83e --- /dev/null +++ b/salt/telegraf/scripts/esindexsize.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +if curl -K /etc/telegraf/elasticsearch.config -s -k -L "https://localhost:9200/" -w "%{http_code}" -o /dev/null | grep -q '200'; then + + DATASTREAM_INFO=$(curl -K /etc/telegraf/elasticsearch.config -s -k -L "https://localhost:9200/_data_stream?format=json") + INDICES=$(curl -K /etc/telegraf/elasticsearch.config -s -k -L "https://localhost:9200/_cat/indices?h=index,store.size&bytes=b&s=index:asc&format=json") + INDICES_WITH_SIZE=() + + while IFS= read -r DS; do + datastream_indices=() + datastream=$(echo "$DS" | jq -r '.name') + # influx doesn't like key starting with '.' + if [[ $datastream != .* ]]; then + while IFS= read -r DS_IDX; do + datastream_indices+=("$DS_IDX") + done < <(echo "$DS" | jq -r '.indices[].index_name') + datastream_size=0 + + for idx in ${datastream_indices[@]}; do + current_index=$(echo "$INDICES" | jq -r --arg index "$idx" '.[] | select(.index == $index)["store.size"]') + datastream_size=$(($datastream_size + $current_index)) + done + INDICES_WITH_SIZE+=("${datastream}=${datastream_size}i") + # echo "$datastream size is $(echo "$datastream_size" | numfmt --to iec)" + fi + done < <(echo "$DATASTREAM_INFO" | jq -c '.data_streams[]') + + measurement="elasticsearch_index_size " + total=${#INDICES_WITH_SIZE[@]} + for idxws in "${!INDICES_WITH_SIZE[@]}"; do + if [[ $idxws -lt $(($total - 1)) ]]; then + measurement+="${INDICES_WITH_SIZE[$idxws]}," + else + measurement+="${INDICES_WITH_SIZE[$idxws]}" + fi + done + + echo "$measurement" + +fi \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 437c44bf8..68189a773 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -17,12 +17,19 @@ base: - schedule - logrotate + # manager node on proper salt version with empty node_data pillar + '( *_manager* or *_eval or *_import or *_standalone ) and G@saltversion:{{saltversion}} and I@node_data:False': + - match: compound + - salt.minion + - salt.master.mine_update_highstate + 'not G@saltversion:{{saltversion}}': - match: compound - salt.minion-state-apply-test - salt.minion - '* and G@saltversion:{{saltversion}}': + # all non managers on the proper salt version + 'not ( *_manager* or *_eval or *_import or *_standalone ) and G@saltversion:{{saltversion}}': - match: compound - salt.minion - patch.os.schedule @@ -33,6 +40,189 @@ base: - docker - docker_clean + # all managers on proper salt version node_data pillar not empty + '( *_manager* or *_eval or *_import or *_standalone ) and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.minion + - patch.os.schedule + - motd + - salt.minion-check + - salt.lasthighstate + - common + - docker + - docker_clean + + '*_eval and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.master + - sensor + - ca + - ssl + - registry + - manager + - backup.config_backup + - nginx + - influxdb + - soc + - kratos + - hydra + - sensoroni + - telegraf + - firewall + - idstools + - suricata.manager + - healthcheck + - elasticsearch + - elastic-fleet-package-registry + - kibana + - pcap + - suricata + - zeek + - strelka + - curator.disabled + - elastalert + - utility + - elasticfleet + + '*_standalone and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.master + - sensor + - ca + - ssl + - registry + - manager + - backup.config_backup + - nginx + - influxdb + - soc + - kratos + - hydra + - firewall + - sensoroni + - telegraf + - idstools + - suricata.manager + - healthcheck + - elasticsearch + - logstash + - redis + - elastic-fleet-package-registry + - kibana + - pcap + - suricata + - zeek + - strelka + - curator.disabled + - elastalert + - utility + - elasticfleet + - stig + - kafka + + '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.master + - ca + - ssl + - registry + - nginx + - influxdb + - strelka.manager + - soc + - kratos + - hydra + - firewall + - manager + - sensoroni + - telegraf + - backup.config_backup + - idstools + - suricata.manager + - elasticsearch + - logstash + - redis + - elastic-fleet-package-registry + - kibana + - curator.disabled + - elastalert + - utility + - elasticfleet + - stig + - kafka + + '*_managersearch and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.master + - ca + - ssl + - registry + - nginx + - influxdb + - strelka.manager + - soc + - kratos + - hydra + - firewall + - manager + - sensoroni + - telegraf + - backup.config_backup + - idstools + - suricata.manager + - elasticsearch + - logstash + - redis + - curator.disabled + - elastic-fleet-package-registry + - kibana + - elastalert + - utility + - elasticfleet + - stig + - kafka + + '*_import and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.master + - sensor + - ca + - ssl + - registry + - manager + - nginx + - influxdb + - strelka.manager + - soc + - kratos + - hydra + - sensoroni + - telegraf + - firewall + - idstools + - suricata.manager + - pcap + - elasticsearch + - elastic-fleet-package-registry + - kibana + - utility + - suricata + - zeek + - elasticfleet + + '*_searchnode and G@saltversion:{{saltversion}}': + - match: compound + - firewall + - ssl + - elasticsearch + - logstash + - sensoroni + - telegraf + - nginx + - elasticfleet.install_agent_grid + - stig + - kafka + '*_sensor and G@saltversion:{{saltversion}}': - match: compound - sensor @@ -49,149 +239,6 @@ base: - elasticfleet.install_agent_grid - stig - '*_eval and G@saltversion:{{saltversion}}': - - match: compound - - salt.master - - sensor - - ca - - ssl - - registry - - manager - - backup.config_backup - - nginx - - influxdb - - soc - - kratos - - hydra - - sensoroni - - telegraf - - firewall - - idstools - - suricata.manager - - healthcheck - - elasticsearch - - elastic-fleet-package-registry - - kibana - - pcap - - suricata - - zeek - - strelka - - curator.disabled - - elastalert - - utility - - elasticfleet - - '*_manager and G@saltversion:{{saltversion}}': - - match: compound - - salt.master - - ca - - ssl - - registry - - nginx - - influxdb - - strelka.manager - - soc - - kratos - - hydra - - firewall - - manager - - sensoroni - - telegraf - - backup.config_backup - - idstools - - suricata.manager - - elasticsearch - - logstash - - redis - - elastic-fleet-package-registry - - kibana - - curator.disabled - - elastalert - - utility - - elasticfleet - - stig - - kafka - - '*_standalone and G@saltversion:{{saltversion}}': - - match: compound - - salt.master - - sensor - - ca - - ssl - - registry - - manager - - backup.config_backup - - nginx - - influxdb - - soc - - kratos - - hydra - - firewall - - sensoroni - - telegraf - - idstools - - suricata.manager - - healthcheck - - elasticsearch - - logstash - - redis - - elastic-fleet-package-registry - - kibana - - pcap - - suricata - - zeek - - strelka - - curator.disabled - - elastalert - - utility - - elasticfleet - - stig - - kafka - - '*_searchnode and G@saltversion:{{saltversion}}': - - match: compound - - firewall - - ssl - - elasticsearch - - logstash - - sensoroni - - telegraf - - nginx - - elasticfleet.install_agent_grid - - stig - - kafka - - '*_managersearch and G@saltversion:{{saltversion}}': - - match: compound - - salt.master - - ca - - ssl - - registry - - nginx - - influxdb - - strelka.manager - - soc - - kratos - - hydra - - firewall - - manager - - sensoroni - - telegraf - - backup.config_backup - - idstools - - suricata.manager - - elasticsearch - - logstash - - redis - - curator.disabled - - elastic-fleet-package-registry - - kibana - - elastalert - - utility - - elasticfleet - - stig - - kafka - '*_heavynode and G@saltversion:{{saltversion}}': - match: compound - sensor @@ -211,34 +258,6 @@ base: - elasticfleet.install_agent_grid - elasticagent - '*_import and G@saltversion:{{saltversion}}': - - match: compound - - salt.master - - sensor - - ca - - ssl - - registry - - manager - - nginx - - influxdb - - strelka.manager - - soc - - kratos - - hydra - - sensoroni - - telegraf - - firewall - - idstools - - suricata.manager - - pcap - - elasticsearch - - elastic-fleet-package-registry - - kibana - - utility - - suricata - - zeek - - elasticfleet - '*_receiver and G@saltversion:{{saltversion}}': - match: compound - ssl diff --git a/setup/files/patch/states/x509_v2.py b/setup/files/patch/states/x509_v2.py deleted file mode 100644 index acc86045f..000000000 --- a/setup/files/patch/states/x509_v2.py +++ /dev/null @@ -1,1814 +0,0 @@ -""" -Manage X.509 certificates -========================= - -.. versionadded:: 3006.0 - - This module represents a complete rewrite of the original ``x509`` modules - and is named ``x509_v2`` since it introduces breaking changes. - -:depends: cryptography - -.. note:: - - All parameters that take a public key, private key, certificate, - CSR or CRL can be specified either as a PEM/hex/base64 string or - a path to a local file encoded in all supported formats for the type. - -Configuration instructions and general remarks are documented -in the :ref:`execution module docs `. - -For the list of breaking changes versus the previous ``x509`` modules, -please also refer to the :ref:`execution module docs `. - -About ------ -This module can enable managing a complete PKI infrastructure, including creating -private keys, CAs, certificates and CRLs. It includes the ability to generate a -private key on a server, and have the corresponding public key sent to a remote -CA to create a CA signed certificate. This can be done in a secure manner, where -private keys are always generated locally and never moved across the network. - -Example -------- -Here is a simple example scenario. In this example ``ca`` is the ca server, -and ``www`` is a web server that needs a certificate signed by ``ca``. - -.. note:: - - Remote signing requires the setup of :term:`Peer Communication` and signing - policies. Please see the :ref:`execution module docs `. - - -/srv/salt/top.sls - -.. code-block:: yaml - - base: - '*': - - cert - 'ca': - - ca - 'www': - - www - -This state creates the CA key, certificate and signing policy. It also publishes -the certificate to the mine, where it can be easily retrieved by other minions. - -.. code-block:: yaml - - # /srv/salt/ca.sls - - Configure the x509 module: - file.managed: - - name: /etc/salt/minion.d/x509.conf - - source: salt://x509.conf - - Restart Salt minion: - cmd.run: - - name: 'salt-call service.restart salt-minion' - - bg: true - - onchanges: - - file: /etc/salt/minion.d/x509.conf - - Ensure PKI directories exist: - file.directory: - - name: /etc/pki/issued_certs - - makedirs: true - - Create CA private key: - x509.private_key_managed: - - name: /etc/pki/ca.key - - keysize: 4096 - - backup: true - - require: - - file: /etc/pki - - Create self-signed CA certificate: - x509.certificate_managed: - - name: /etc/pki/ca.crt - - signing_private_key: /etc/pki/ca.key - - CN: ca.example.com - - C: US - - ST: Utah - - L: Salt Lake City - - basicConstraints: "critical, CA:true" - - keyUsage: "critical, cRLSign, keyCertSign" - - subjectKeyIdentifier: hash - - authorityKeyIdentifier: keyid:always,issuer - - days_valid: 3650 - - days_remaining: 0 - - backup: true - - require: - - x509: /etc/pki/ca.key - -.. code-block:: yaml - - # /srv/salt/x509.conf - - # enable x509_v2 - features: - x509_v2: true - - # publish the CA certificate to the mine - mine_functions: - x509.get_pem_entries: [/etc/pki/ca.crt] - - # define at least one signing policy for remote signing - x509_signing_policies: - www: - - minions: 'www' - - signing_private_key: /etc/pki/ca.key - - signing_cert: /etc/pki/ca.crt - - C: US - - ST: Utah - - L: Salt Lake City - - basicConstraints: "critical CA:false" - - keyUsage: "critical keyEncipherment" - - subjectKeyIdentifier: hash - - authorityKeyIdentifier: keyid:always,issuer - - days_valid: 30 - - copypath: /etc/pki/issued_certs/ - - -This example state will instruct all minions to trust certificates signed by -our new CA. Mind that this example works for Debian-based OS only. -Also note the Jinja call to encode the string to JSON, which will avoid -YAML issues with newline characters. - -.. code-block:: jinja - - # /srv/salt/cert.sls - - Ensure the CA trust bundle exists: - file.directory: - - name: /usr/local/share/ca-certificates - - Ensure our self-signed CA certificate is included: - x509.pem_managed: - - name: /usr/local/share/ca-certificates/myca.crt - - text: {{ salt["mine.get"]("ca", "x509.get_pem_entries")["ca"]["/etc/pki/ca.crt"] | json }} - -This state creates a private key, then requests a certificate signed by our CA -according to the www policy. - -.. code-block:: yaml - - # /srv/salt/www.sls - - Ensure PKI directory exists: - file.directory: - - name: /etc/pki - - Create private key for the certificate: - x509.private_key_managed: - - name: /etc/pki/www.key - - keysize: 4096 - - backup: true - - require: - - file: /etc/pki - - Request certificate: - x509.certificate_managed: - - name: /etc/pki/www.crt - - ca_server: ca - - signing_policy: www - - private_key: /etc/pki/www.key - - CN: www.example.com - - days_remaining: 7 - - backup: true - - require: - - x509: /etc/pki/www.key -""" -import base64 -import copy -import datetime -import logging -import os.path - -import salt.utils.context -import salt.utils.files -from salt.exceptions import CommandExecutionError, SaltInvocationError -from salt.features import features -from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS - -try: - import cryptography.x509 as cx509 - from cryptography.exceptions import UnsupportedAlgorithm - from cryptography.hazmat.primitives import hashes - - import salt.utils.x509 as x509util - - HAS_CRYPTOGRAPHY = True -except ImportError: - HAS_CRYPTOGRAPHY = False - - -log = logging.getLogger(__name__) - -__virtualname__ = "x509" - - -def __virtual__(): - if not HAS_CRYPTOGRAPHY: - return (False, "Could not load cryptography") - if not features.get("x509_v2"): - return ( - False, - "x509_v2 needs to be explicitly enabled by setting `x509_v2: true` " - "in the minion configuration value `features` until Salt 3008 (Argon).", - ) - return __virtualname__ - - -def certificate_managed( - name, - days_remaining=None, - ca_server=None, - signing_policy=None, - encoding="pem", - append_certs=None, - copypath=None, - prepend_cn=False, - digest="sha256", - signing_private_key=None, - signing_private_key_passphrase=None, - signing_cert=None, - public_key=None, - private_key=None, - private_key_passphrase=None, - csr=None, - subject=None, - serial_number=None, - not_before=None, - not_after=None, - days_valid=None, - pkcs12_passphrase=None, - pkcs12_encryption_compat=False, - pkcs12_friendlyname=None, - **kwargs, -): - """ - Ensure an X.509 certificate is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_certificate `, - as well as most ones for `:py:func:`file.managed `. - - name - The path the certificate should be present at. - - days_remaining - The certificate will be recreated once the remaining certificate validity - period is less than this number of days. - Defaults to ``90`` (until v3009) or ``7`` (from v3009 onwards). - - ca_server - Request a remotely signed certificate from ca_server. For this to - work, a ``signing_policy`` must be specified, and that same policy - must be configured on the ca_server. Also, the Salt master must - permit peers to call the ``x509.sign_remote_certificate`` function. - See the :ref:`execution module docs ` for details. - - signing_policy - The name of a configured signing policy. Parameters specified in there - are hardcoded and cannot be overridden. This is required for remote signing, - otherwise optional. - - encoding - Specify the encoding of the resulting certificate. It can be serialized - as a ``pem`` (or ``pkcs7_pem``) text file or in several binary formats - (``der``, ``pkcs7_der``, ``pkcs12``). Defaults to ``pem``. - - append_certs - A list of additional certificates to append to the new one, e.g. to create a CA chain. - - .. note:: - - Mind that when ``der`` encoding is in use, appending certificatees is prohibited. - - copypath - Create a copy of the issued certificate in PEM format in this directory. - The file will be named ``.crt`` if prepend_cn is false. - - prepend_cn - When ``copypath`` is set, prepend the common name of the certificate to - the file name like so: ``-.crt``. Defaults to false. - - digest - The hashing algorithm to use for the signature. Valid values are: - sha1, sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, - sha3_256, sha3_384, sha3_512. Defaults to ``sha256``. - This will be ignored for ``ed25519`` and ``ed448`` key types. - - signing_private_key - The private key corresponding to the public key in ``signing_cert``. Required. - - signing_private_key_passphrase - If ``signing_private_key`` is encrypted, the passphrase to decrypt it. - - signing_cert - The CA certificate to be used for signing the issued certificate. - - public_key - The public key the certificate should be issued for. Other ways of passing - the required information are ``private_key`` and ``csr``. If neither are set, - the public key of the ``signing_private_key`` will be included, i.e. - a self-signed certificate is generated. - - private_key - The private key corresponding to the public key the certificate should - be issued for. This is one way of specifying the public key that will - be included in the certificate, the other ones being ``public_key`` and ``csr``. - - private_key_passphrase - If ``private_key`` is specified and encrypted, the passphrase to decrypt it. - - csr - A certificate signing request to use as a base for generating the certificate. - The following information will be respected, depending on configuration: - - * public key - * extensions, if not otherwise specified (arguments, signing_policy) - - subject - The subject's distinguished name embedded in the certificate. This is one way of - passing this information (see ``kwargs`` below for the other). - This argument will be preferred and allows to control the order of RDNs in the DN - as well as to embed RDNs with multiple attributes. - This can be specified as a RFC4514-encoded string (``CN=example.com,O=Example Inc,C=US``, - mind that the rendered order is reversed from what is embedded), a list - of RDNs encoded as in RFC4514 (``["C=US", "O=Example Inc", "CN=example.com"]``) - or a dictionary (``{"CN": "example.com", "C": "US", "O": "Example Inc"}``, - default ordering). - Multiple name attributes per RDN are concatenated with a ``+``. - - .. note:: - - Parsing of RFC4514 strings requires at least cryptography release 37. - - serial_number - A serial number to be embedded in the certificate. If unspecified, will - autogenerate one. This should be an integer, either in decimal or - hexadecimal notation. - - not_before - Set a specific date the certificate should not be valid before. - The format should follow ``%Y-%m-%d %H:%M:%S`` and will be interpreted as GMT/UTC. - Defaults to the time of issuance. - - not_after - Set a specific date the certificate should not be valid after. - The format should follow ``%Y-%m-%d %H:%M:%S`` and will be interpreted as GMT/UTC. - If unspecified, defaults to the current time plus ``days_valid`` days. - - days_valid - If ``not_after`` is unspecified, the number of days from the time of issuance - the certificate should be valid for. - Defaults to ``365`` (until v3009) or ``30`` (from v3009 onwards). - - pkcs12_passphrase - When encoding a certificate as ``pkcs12``, encrypt it with this passphrase. - - .. note:: - - PKCS12 encryption is very weak and `should not be relied on for security `_. - - pkcs12_encryption_compat - OpenSSL 3 and cryptography v37 switched to a much more secure default - encryption for PKCS12, which might be incompatible with some systems. - This forces the legacy encryption. Defaults to False. - - pkcs12_friendlyname - When encoding a certificate as ``pkcs12``, a name for the certificate can be included. - - kwargs - Embedded X.509v3 extensions and the subject's distinguished name can be - controlled via supplemental keyword arguments. See - :py:func:`x509.create_certificate ` - for an overview. - """ - # Deprecation checks vs the old x509 module - if days_valid is None and not_after is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_valid` will change to 30. Please adapt your code accordingly.", - ) - days_valid = 365 - except RuntimeError: - days_valid = 30 - - if days_remaining is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_remaining` will change to 7. Please adapt your code accordingly.", - ) - days_remaining = 90 - except RuntimeError: - days_remaining = 7 - - if "algorithm" in kwargs: - salt.utils.versions.warn_until( - "Potassium", - "`algorithm` has been renamed to `digest`. Please update your code.", - ) - digest = kwargs.pop("algorithm") - kwargs = x509util.ensure_cert_kwargs_compat(kwargs) - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The certificate is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, cert_args = _split_file_kwargs(_filter_state_internal_kwargs(kwargs)) - append_certs = append_certs or [] - if not isinstance(append_certs, list): - append_certs = [append_certs] - - try: - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - if __salt__["file.file_exists"](real_name): - try: - ( - current, - current_encoding, - current_chain, - current_extra, - ) = x509util.load_cert( - real_name, passphrase=pkcs12_passphrase, get_encoding=True - ) - except SaltInvocationError as err: - if "Bad decrypt" in str(err): - changes["pkcs12_passphrase"] = True - elif any( - ( - "Could not deserialize binary data" in str(err), - "Could not load PEM-encoded" in str(err), - ) - ): - replace = True - else: - raise - else: - if encoding != current_encoding: - changes["encoding"] = encoding - elif encoding == "pkcs12" and current_extra.cert.friendly_name != ( - salt.utils.stringutils.to_bytes(pkcs12_friendlyname) - if pkcs12_friendlyname - else None - ): - changes["pkcs12_friendlyname"] = pkcs12_friendlyname - - if ( - current.not_valid_after - < datetime.datetime.utcnow() - + datetime.timedelta(days=days_remaining) - ): - changes["expiration"] = True - - current_chain = current_chain or [] - ca_chain = [x509util.load_cert(x) for x in append_certs] - if not _compare_ca_chain(current_chain, ca_chain): - changes["additional_certs"] = True - - ( - builder, - private_key_loaded, - signing_cert_loaded, - final_kwargs, - ) = _build_cert( - ca_server=ca_server, - signing_policy=signing_policy, - digest=digest, # passed because of signing_policy merging - signing_private_key=signing_private_key, - signing_private_key_passphrase=signing_private_key_passphrase, - signing_cert=signing_cert, - public_key=public_key, - private_key=private_key, - private_key_passphrase=private_key_passphrase, - csr=csr, - subject=subject, - serial_number=serial_number, - not_before=not_before, - not_after=not_after, - days_valid=days_valid, - **cert_args, - ) - - try: - if current.signature_hash_algorithm is not None and not isinstance( - current.signature_hash_algorithm, - type(x509util.get_hashing_algorithm(final_kwargs["digest"])), - ): - # ed25519, ed448 do not use a separate hash for signatures, hence algo is None - changes["digest"] = digest - except UnsupportedAlgorithm: - # this eg happens with sha3 in cryptography < v39 - log.warning( - "Could not determine signature hash algorithm of '%s'. " - "Continuing anyways", - name, - ) - - changes.update( - _compare_cert( - current, - builder, - signing_cert=signing_cert_loaded, - serial_number=serial_number, - not_before=not_before, - not_after=not_after, - ) - ) - else: - changes["created"] = name - - if replace: - changes["replaced"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if current and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The certificate would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - { - "additional_certs", - "encoding", - "pkcs12_friendlyname", - }: - # do not reissue if only metaparameters changed - if encoding == "pkcs12": - cert = __salt__["x509.encode_certificate"]( - current, - append_certs=append_certs, - encoding=encoding, - private_key=private_key_loaded, - pkcs12_passphrase=pkcs12_passphrase, - pkcs12_encryption_compat=pkcs12_encryption_compat, - pkcs12_friendlyname=pkcs12_friendlyname, - ) - else: - cert = __salt__["x509.encode_certificate"]( - current, encoding=encoding, append_certs=append_certs - ) - else: - # request a new certificate otherwise - cert = __salt__["x509.create_certificate"]( - ca_server=ca_server, - signing_policy=signing_policy, - encoding=encoding, - append_certs=append_certs, - pkcs12_passphrase=pkcs12_passphrase, - pkcs12_encryption_compat=pkcs12_encryption_compat, - pkcs12_friendlyname=pkcs12_friendlyname, - digest=digest, - signing_private_key=signing_private_key, - signing_private_key_passphrase=signing_private_key_passphrase, - signing_cert=signing_cert, - public_key=public_key, - private_key=private_key, - private_key_passphrase=private_key_passphrase, - csr=csr, - subject=subject, - serial_number=serial_number, - not_before=not_before, - not_after=not_after, - days_valid=days_valid, - **cert_args, - ) - ret["comment"] = f"The certificate has been {verb}d" - if encoding not in ["pem", "pkcs7_pem"]: - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(cert), file_args.get("backup", "") - ) - - if not changes or encoding in ["pem", "pkcs7_pem"]: - replace = bool(encoding in ["pem", "pkcs7_pem"] and changes) - contents = cert if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def crl_managed( - name, - signing_private_key, - revoked, - days_remaining=None, - signing_cert=None, - signing_private_key_passphrase=None, - include_expired=False, - days_valid=None, - digest="sha256", - encoding="pem", - extensions=None, - **kwargs, -): - """ - Ensure a certificate revocation list is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_crl `, - as well as most ones for `:py:func:`file.managed `. - - name - The path the certificate revocation list should be present at. - - signing_private_key - Your certificate authority's private key. It will be used to sign - the CRL. Required. - - revoked - A list of dicts containing all the certificates to revoke. Each dict - represents one certificate. A dict must contain either the key - ``serial_number`` with the value of the serial number to revoke, or - ``certificate`` with some reference to the certificate to revoke. - - The dict can optionally contain the ``revocation_date`` key. If this - key is omitted, the revocation date will be set to now. It should be a - string in the format ``%Y-%m-%d %H:%M:%S``. - - The dict can also optionally contain the ``not_after`` key. This is - redundant if the ``certificate`` key is included. If the - ``certificate`` key is not included, this can be used for the logic - behind the ``include_expired`` parameter. It should be a string in - the format ``%Y-%m-%d %H:%M:%S``. - - The dict can also optionally contain the ``extensions`` key, which - allows to set CRL entry-specific extensions. The following extensions - are supported: - - certificateIssuer - Identifies the certificate issuer associated with an entry in an - indirect CRL. The format is the same as for subjectAltName. - - CRLReason - Identifies the reason for certificate revocation. - Available choices are ``unspecified``, ``keyCompromise``, ``CACompromise``, - ``affiliationChanged``, ``superseded``, ``cessationOfOperation``, - ``certificateHold``, ``privilegeWithdrawn``, ``aACompromise`` and - ``removeFromCRL``. - - invalidityDate - Provides the date on which the certificate became invalid. - The value should be a string in the same format as ``revocation_date``. - - days_remaining - The certificate revocation list will be recreated once the remaining - CRL validity period is less than this number of days. - Defaults to ``30`` (until v3009) or ``3`` (from v3009 onwards). - Set to 0 to disable automatic renewal without anything changing. - - signing_cert - The CA certificate to be used for signing the issued certificate. - - signing_private_key_passphrase - If ``signing_private_key`` is encrypted, the passphrase to decrypt it. - - include_expired - Also include already expired certificates in the CRL. Defaults to false. - - days_valid - The number of days that the CRL should be valid for. This sets the ``Next Update`` - field in the CRL. Defaults to ``100`` (until v3009) or ``7`` (from v3009 onwards). - - digest - The hashing algorithm to use for the signature. Valid values are: - sha1, sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, - sha3_256, sha3_384, sha3_512. Defaults to ``sha256``. - This will be ignored for ``ed25519`` and ``ed448`` key types. - - encoding - Specify the encoding of the resulting certificate revocation list. - It can be serialized as a ``pem`` text or binary ``der`` file. - Defaults to ``pem``. - - extensions - Add CRL extensions. See :py:func:`x509.create_crl ` - for details. - - .. note:: - - For ``cRLNumber``, in addition the value ``auto`` is supported, which - automatically increases the counter every time a new CRL is issued. - - Example: - - .. code-block:: yaml - - Manage CRL: - x509.crl_managed: - - name: /etc/pki/ca.crl - - signing_private_key: /etc/pki/myca.key - - signing_cert: /etc/pki/myca.crt - - revoked: - - certificate: /etc/pki/certs/badweb.crt - revocation_date: 2022-11-01 00:00:00 - extensions: - CRLReason: keyCompromise - - serial_number: D6:D2:DC:D8:4D:5C:C0:F4 - not_after: 2023-03-14 00:00:00 - revocation_date: 2022-10-25 00:00:00 - extensions: - CRLReason: cessationOfOperation - - extensions: - cRLNumber: auto - """ - if "text" in kwargs: - salt.utils.versions.kwargs_warn_until(["text"], "Potassium") - kwargs.pop("text") - - if days_valid is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_valid` will change to 7. Please adapt your code accordingly.", - ) - days_valid = 100 - except RuntimeError: - days_valid = 7 - - if days_remaining is None: - try: - salt.utils.versions.warn_until( - "Potassium", - "The default value for `days_remaining` will change to 3. Please adapt your code accordingly.", - ) - days_remaining = 30 - except RuntimeError: - days_remaining = 3 - - revoked_parsed = [] - for rev in revoked: - parsed = {} - if len(rev) == 1 and isinstance(rev[next(iter(rev))], list): - salt.utils.versions.warn_until( - "Potassium", - "Revoked certificates should be specified as a simple list of dicts.", - ) - for val in rev[next(iter(rev))]: - parsed.update(val) - if "reason" in (parsed or rev): - salt.utils.versions.warn_until( - "Potassium", - "The `reason` parameter for revoked certificates should be specified in extensions:CRLReason.", - ) - salt.utils.dictupdate.set_dict_key_value( - (parsed or rev), "extensions:CRLReason", (parsed or rev).pop("reason") - ) - revoked_parsed.append(parsed or rev) - revoked = revoked_parsed - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The certificate revocation list is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, extra_args = _split_file_kwargs(_filter_state_internal_kwargs(kwargs)) - extensions = extensions or {} - if extra_args: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(extra_args)}") - - try: - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - if __salt__["file.file_exists"](real_name): - try: - current, current_encoding = x509util.load_crl( - real_name, get_encoding=True - ) - except SaltInvocationError as err: - if any( - ( - "Could not load PEM-encoded" in str(err), - "Could not load DER-encoded" in str(err), - ) - ): - replace = True - else: - raise - else: - try: - if current.signature_hash_algorithm is not None and not isinstance( - current.signature_hash_algorithm, - type(x509util.get_hashing_algorithm(digest)), - ): - # ed25519, ed448 do not use a separate hash for signatures, hence algo is None - # although CA certificates should not be using those currently - changes["digest"] = digest - except UnsupportedAlgorithm: - # this eg happens with sha3 digest in cryptography < v39 - log.warning( - "Could not determine signature hash algorithm of '%s'. " - "Continuing anyways", - name, - ) - - if encoding != current_encoding: - changes["encoding"] = encoding - if days_remaining and ( - current.next_update - < datetime.datetime.utcnow() - + datetime.timedelta(days=days_remaining) - ): - changes["expiration"] = True - - # "auto" is a value that is managed in this function and cannot not be compared - crl_auto = extensions.get("cRLNumber") == "auto" - if crl_auto: - extensions.pop("cRLNumber") - - builder, sig_privkey = x509util.build_crl( - signing_private_key, - revoked, - signing_cert=signing_cert, - signing_private_key_passphrase=signing_private_key_passphrase, - include_expired=include_expired, - days_valid=days_valid, - extensions=extensions, - ) - changes.update(_compare_crl(current, builder, sig_privkey.public_key())) - if crl_auto: - # put cRLNumber = auto back if it was set - extensions["cRLNumber"] = "auto" - changes["extensions"]["removed"].pop( - changes["extensions"]["removed"].index("cRLNumber") - ) - if not any(changes["extensions"].values()): - changes.pop("extensions") - else: - changes["created"] = name - - if replace: - changes["replaced"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if current and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The certificate revocation list would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - {"encoding"}: - # do not regenerate if only metaparameters changed - crl = __salt__["x509.encode_crl"](current, encoding=encoding) - else: - # autoincrease cRLNumber counter, if requested - if extensions.get("cRLNumber") == "auto": - try: - extensions["cRLNumber"] = ( - current.extensions.get_extension_for_class( - cx509.CRLNumber - ).value.crl_number - + 1 - ) - except (AttributeError, cx509.ExtensionNotFound): - extensions["cRLNumber"] = 1 - crl = __salt__["x509.create_crl"]( - signing_private_key, - revoked, - signing_cert=signing_cert, - signing_private_key_passphrase=signing_private_key_passphrase, - include_expired=include_expired, - days_valid=days_valid, - digest=digest, - encoding=encoding, - extensions=extensions, - ) - ret["comment"] = f"The certificate revocation list has been {verb}d" - if encoding == "der": - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(crl), file_args.get("backup", "") - ) - - if not changes or encoding == "pem": - replace = bool((encoding == "pem") and changes) - contents = crl if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def csr_managed( - name, - private_key, - private_key_passphrase=None, - digest="sha256", - encoding="pem", - subject=None, - **kwargs, -): - """ - Ensure a certificate signing request is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_csr `, - as well as most ones for :py:func:`file.managed `. - - name - The path the certificate signing request should be present at. - - private_key - The private key corresponding to the public key the certificate should - be issued for. The CSR will be signed by it. Required. - - private_key_passphrase - If ``private_key`` is encrypted, the passphrase to decrypt it. - - digest - The hashing algorithm to use for the signature. Valid values are: - sha1, sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, - sha3_256, sha3_384, sha3_512. Defaults to ``sha256``. - This will be ignored for ``ed25519`` and ``ed448`` key types. - - encoding - Specify the encoding of the resulting certificate revocation list. - It can be serialized as a ``pem`` text or binary ``der`` file. - Defaults to ``pem``. - - kwargs - Embedded X.509v3 extensions and the subject's distinguished name can be - controlled via supplemental keyword arguments. - See :py:func:`x509.create_certificate ` - for an overview. Mind that some extensions are not available for CSR - (``authorityInfoAccess``, ``authorityKeyIdentifier``, - ``issuerAltName``, ``crlDistributionPoints``). - """ - # Deprecation checks vs the old x509 module - if "algorithm" in kwargs: - salt.utils.versions.warn_until( - "Potassium", - "`algorithm` has been renamed to `digest`. Please update your code.", - ) - digest = kwargs.pop("algorithm") - kwargs = x509util.ensure_cert_kwargs_compat(kwargs) - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The certificate signing request is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, csr_args = _split_file_kwargs(_filter_state_internal_kwargs(kwargs)) - - try: - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - if __salt__["file.file_exists"](real_name): - try: - current, current_encoding = x509util.load_csr( - real_name, get_encoding=True - ) - except SaltInvocationError as err: - if any( - ( - "Could not load PEM-encoded" in str(err), - "Could not load DER-encoded" in str(err), - ) - ): - replace = True - else: - raise - except cx509.InvalidVersion: - # by default, the previous x509 modules generated CSR with - # invalid versions, which leads to an exception in cryptography >= v38 - changes["invalid_version"] = True - replace = True - else: - try: - if current.signature_hash_algorithm is not None and not isinstance( - current.signature_hash_algorithm, - type(x509util.get_hashing_algorithm(digest)), - ): - # ed25519, ed448 do not use a separate hash for signatures, hence algo is None - changes["digest"] = digest - except UnsupportedAlgorithm: - # this eg happens with sha3 digest in cryptography < v39 - log.warning( - "Could not determine signature hash algorithm of '%s'. " - "Continuing anyways", - name, - ) - - if encoding != current_encoding: - changes["encoding"] = encoding - - builder, privkey = x509util.build_csr( - private_key, - private_key_passphrase=private_key_passphrase, - subject=subject, - **csr_args, - ) - if not x509util.is_pair(current.public_key(), privkey): - changes["private_key"] = True - changes.update(_compare_csr(current, builder)) - else: - changes["created"] = name - - if replace: - changes["replaced"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if current and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The certificate signing request would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - {"encoding"}: - # do not regenerate if only metaparameters changed - csr = __salt__["x509.encode_csr"](current, encoding=encoding) - else: - csr = __salt__["x509.create_csr"]( - private_key, - private_key_passphrase=private_key_passphrase, - digest=digest, - encoding=encoding, - subject=subject, - **csr_args, - ) - ret["comment"] = f"The certificate signing request has been {verb}d" - if encoding == "der": - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(csr), file_args.get("backup", "") - ) - if not changes or encoding == "pem": - replace = bool((encoding == "pem") and changes) - contents = csr if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def pem_managed(name, text, **kwargs): - """ - Manage the contents of a PEM file directly with the content in text, - ensuring correct formatting. - - name - The path to the file to manage. - - text - The PEM-formatted text to write. - - kwargs - Most arguments supported by :py:func:`file.managed ` are passed through. - """ - file_args, extra_args = _split_file_kwargs(kwargs) - if extra_args: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(extra_args)}") - - try: - file_args["contents"] = __salt__["x509.get_pem_entry"](text=text) - except (CommandExecutionError, SaltInvocationError) as err: - return {"name": name, "result": False, "comment": str(err), "changes": {}} - return _file_managed(name, **file_args) - - -def private_key_managed( - name, - algo="rsa", - keysize=None, - passphrase=None, - encoding="pem", - new=False, - overwrite=False, - pkcs12_encryption_compat=False, - **kwargs, -): - """ - Ensure a private key is present as specified. - - This function accepts the same arguments as :py:func:`x509.create_private_key `, - as well as most ones for :py:func:`file.managed `. - - .. note:: - - If ``mode`` is unspecified, it will default to ``0400``. - - name - The path the private key should be present at. - - algo - The digital signature scheme the private key should be based on. - Available: ``rsa``, ``ec``, ``ed25519``, ``ed448``. Defaults to ``rsa``. - - keysize - For ``rsa``, specifies the bitlength of the private key (2048, 3072, 4096). - For ``ec``, specifies the NIST curve to use (256, 384, 521). - Irrelevant for Edwards-curve schemes (`ed25519``, ``ed448``). - Defaults to 2048 for RSA and 256 for EC. - - passphrase - If this is specified, the private key will be encrypted using this - passphrase. The encryption algorithm cannot be selected, it will be - determined automatically as the best available one. - - encoding - Specify the encoding of the resulting private key. It can be serialized - as a ``pem`` text, binary ``der`` or ``pkcs12`` file. - Defaults to ``pem``. - - new - Always create a new key. Defaults to false. - Combining new with :mod:`prereq ` - can allow key rotation whenever a new certificate is generated. - - overwrite - Overwrite an existing private key if the provided passphrase cannot decrypt it. - Defaults to false. - - pkcs12_encryption_compat - Some operating systems are incompatible with the encryption defaults - for PKCS12 used since OpenSSL v3. This switch triggers a fallback to - ``PBESv1SHA1And3KeyTripleDESCBC``. - Please consider the `notes on PKCS12 encryption `_. - - Example: - - The Jinja templating in this example ensures a new private key is generated - if the file does not exist and whenever the associated certificate - is to be renewed. - - .. code-block:: jinja - - Manage www private key: - x509.private_key_managed: - - name: /etc/pki/www.key - - keysize: 4096 - - new: true - {%- if salt["file.file_exists"]("/etc/pki/www.key") %} - - prereq: - - x509: /etc/pki/www.crt - {%- endif %} - """ - # Deprecation checks vs the old x509 module - if "bits" in kwargs: - salt.utils.versions.warn_until( - "Potassium", - "`bits` has been renamed to `keysize`. Please update your code.", - ) - keysize = kwargs.pop("bits") - - ignored_params = {"cipher", "verbose", "text"}.intersection( - kwargs - ) # path, overwrite - if ignored_params: - salt.utils.versions.kwargs_warn_until(ignored_params, "Potassium") - for x in ignored_params: - kwargs.pop(x) - - ret = { - "name": name, - "changes": {}, - "result": True, - "comment": "The private key is in the correct state", - } - current = current_encoding = None - changes = {} - verb = "create" - file_args, extra_args = _split_file_kwargs(kwargs) - - if extra_args: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(extra_args)}") - - if not file_args.get("mode"): - # ensure secure defaults - file_args["mode"] = "0400" - - try: - if keysize and algo in ["ed25519", "ed448"]: - raise SaltInvocationError(f"keysize is an invalid parameter for {algo}") - - # check file.managed changes early to avoid using unnecessary resources - file_managed_test = _file_managed(name, test=True, replace=False, **file_args) - - if file_managed_test["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = "Problem while testing file.managed changes, see its output" - _add_sub_state_run(ret, file_managed_test) - return ret - - if "is not present and is not set for creation" in file_managed_test["comment"]: - _add_sub_state_run(ret, file_managed_test) - return ret - - real_name = name - replace = False - - # handle follow_symlinks - if __salt__["file.is_link"](name): - if file_args.get("follow_symlinks", True): - real_name = os.path.realpath(name) - else: - # workaround https://github.com/saltstack/salt/issues/31802 - __salt__["file.remove"](name) - replace = True - - file_exists = __salt__["file.file_exists"](real_name) - - if file_exists and not new: - try: - current, current_encoding, _ = x509util.load_privkey( - real_name, passphrase=passphrase, get_encoding=True - ) - except SaltInvocationError as err: - if "Bad decrypt" in str(err): - if not overwrite: - raise CommandExecutionError( - "The provided passphrase cannot decrypt the private key. " - "Pass overwrite: true to force regeneration" - ) from err - changes["passphrase"] = True - elif any( - ( - "Could not deserialize binary data" in str(err), - "Could not load DER-encoded" in str(err), - "Could not load PEM-encoded" in str(err), - ) - ): - if not overwrite: - raise CommandExecutionError( - "The existing file does not seem to be a private key " - "formatted as DER, PEM or embedded in PKCS12. " - "Pass overwrite: true to force regeneration" - ) from err - replace = True - elif "Private key is unencrypted" in str(err): - changes["passphrase"] = True - current, current_encoding, _ = x509util.load_privkey( - real_name, passphrase=None, get_encoding=True - ) - elif "Private key is encrypted" in str(err) and not passphrase: - if not overwrite: - raise CommandExecutionError( - "The existing file is encrypted. Pass overwrite: true " - "to force regeneration without passphrase" - ) from err - changes["passphrase"] = True - else: - raise - if current: - key_type = x509util.get_key_type(current) - check_keysize = keysize - if check_keysize is None: - if algo == "rsa": - check_keysize = 2048 - elif algo == "ec": - check_keysize = 256 - if any( - ( - (algo == "rsa" and not key_type == x509util.KEY_TYPE.RSA), - (algo == "ec" and not key_type == x509util.KEY_TYPE.EC), - (algo == "ed25519" and not key_type == x509util.KEY_TYPE.ED25519), - (algo == "ed448" and not key_type == x509util.KEY_TYPE.ED448), - ) - ): - changes["algo"] = algo - if ( - "algo" not in changes - and algo in ("rsa", "ec") - and current.key_size != check_keysize - ): - changes["keysize"] = keysize - if encoding != current_encoding: - changes["encoding"] = encoding - elif file_exists and new: - changes["replaced"] = name - else: - changes["created"] = name - - if ( - not changes - and file_managed_test["result"] - and not file_managed_test["changes"] - ): - _add_sub_state_run(ret, file_managed_test) - return ret - - ret["changes"] = changes - if file_exists and changes: - verb = "recreate" - - if __opts__["test"]: - ret["result"] = None if changes else True - ret["comment"] = ( - f"The private key would have been {verb}d" - if changes - else ret["comment"] - ) - _add_sub_state_run(ret, file_managed_test) - return ret - - if changes: - if not set(changes) - {"encoding", "passphrase"}: - # do not regenerate if only metaparameters changed - pk = __salt__["x509.encode_private_key"]( - current, passphrase=passphrase, encoding=encoding - ) - else: - pk = __salt__["x509.create_private_key"]( - algo=algo, - keysize=keysize, - passphrase=passphrase, - encoding=encoding, - pkcs12_encryption_compat=pkcs12_encryption_compat, - ) - ret["comment"] = f"The private key has been {verb}d" - if encoding != "pem": - # file.managed does not support binary contents, so create - # an empty file first (makedirs). This will not work with check_cmd! - file_managed_ret = _file_managed(name, replace=False, **file_args) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - _safe_atomic_write( - real_name, base64.b64decode(pk), file_args.get("backup", "") - ) - - if not changes or encoding == "pem": - replace = bool((encoding == "pem") and changes) - contents = pk if replace else None - file_managed_ret = _file_managed( - name, contents=contents, replace=replace, **file_args - ) - _add_sub_state_run(ret, file_managed_ret) - if not _check_file_ret(file_managed_ret, ret, current): - return ret - except (CommandExecutionError, SaltInvocationError) as err: - ret["result"] = False - ret["comment"] = str(err) - ret["changes"] = {} - return ret - - -def _filter_state_internal_kwargs(kwargs): - # check_cmd is a valid argument to file.managed - ignore = set(_STATE_INTERNAL_KEYWORDS) - {"check_cmd"} - return {k: v for k, v in kwargs.items() if k not in ignore} - - -def _split_file_kwargs(kwargs): - valid_file_args = [ - "user", - "group", - "mode", - "attrs", - "makedirs", - "dir_mode", - "backup", - "create", - "follow_symlinks", - "check_cmd", - "tmp_dir", - "tmp_ext", - "selinux", - "encoding", - "encoding_errors", - "win_owner", - "win_perms", - "win_deny_perms", - "win_inheritance", - "win_perms_reset", - ] - file_args = {"show_changes": False} - extra_args = {} - for k, v in kwargs.items(): - if k in valid_file_args: - file_args[k] = v - else: - extra_args[k] = v - return file_args, extra_args - - -def _add_sub_state_run(ret, sub): - sub["low"] = { - "name": ret["name"], - "state": "file", - "__id__": __low__["__id__"], - "fun": "managed", - } - if "sub_state_run" not in ret: - ret["sub_state_run"] = [] - ret["sub_state_run"].append(sub) - - -def _file_managed(name, test=None, **kwargs): - if test not in [None, True]: - raise SaltInvocationError("test param can only be None or True") - # work around https://github.com/saltstack/salt/issues/62590 - test = test or __opts__["test"] - file_managed = __states__["file.managed"] - if test: - # calls via __salt__["state.single"](..., test=test) - # can overwrite __opts__["test"] permanently. Workaround: - opts = __opts__ - if not __opts__["test"]: - opts = copy.copy(__opts__) - opts["test"] = test - with salt.utils.context.func_globals_inject(file_managed, __opts__=opts): - # The file execution module accesses __opts__["test"] as well - with salt.utils.context.func_globals_inject( - __salt__["file.check_perms"], __opts__=opts - ): - with salt.utils.context.func_globals_inject( - __salt__["file.manage_file"], __opts__=opts - ): - return file_managed(name, **kwargs) - return file_managed(name, **kwargs) - - -def _check_file_ret(fret, ret, current): - if fret["result"] is False: - ret["result"] = False - ret[ - "comment" - ] = f"Could not {'create' if not current else 'update'} file, see file.managed output" - ret["changes"] = {} - return False - return True - - -def _build_cert( - ca_server=None, signing_policy=None, signing_private_key=None, **kwargs -): - final_kwargs = copy.deepcopy(kwargs) - x509util.merge_signing_policy( - __salt__["x509.get_signing_policy"](signing_policy, ca_server=ca_server), - final_kwargs, - ) - - builder, _, private_key_loaded, signing_cert = x509util.build_crt( - signing_private_key, - skip_load_signing_private_key=ca_server is not None, - **final_kwargs, - ) - return builder, private_key_loaded, signing_cert, final_kwargs - - -def _compare_cert(current, builder, signing_cert, serial_number, not_before, not_after): - changes = {} - - if ( - serial_number is not None - and _getattr_safe(builder, "_serial_number") != current.serial_number - ): - changes["serial_number"] = serial_number - - if not x509util.match_pubkey( - _getattr_safe(builder, "_public_key"), current.public_key() - ): - changes["private_key"] = True - - if signing_cert and not x509util.verify_signature( - current, signing_cert.public_key() - ): - changes["signing_private_key"] = True - - if _getattr_safe(builder, "_subject_name") != current.subject: - changes["subject_name"] = _getattr_safe( - builder, "_subject_name" - ).rfc4514_string() - - if _getattr_safe(builder, "_issuer_name") != current.issuer: - changes["issuer_name"] = _getattr_safe(builder, "_issuer_name").rfc4514_string() - - ext_changes = _compare_exts(current, builder) - if any(ext_changes.values()): - changes["extensions"] = ext_changes - return changes - - -def _compare_csr(current, builder): - changes = {} - - # if _getattr_safe(builder, "_subject_name") != current.subject: - if not _compareattr_safe(builder, "_subject_name", current.subject): - changes["subject_name"] = _getattr_safe( - builder, "_subject_name" - ).rfc4514_string() - - ext_changes = _compare_exts(current, builder) - if any(ext_changes.values()): - changes["extensions"] = ext_changes - return changes - - -def _compare_crl(current, builder, sig_pubkey): - # these are necessary because the classes do not have the required method - def _get_revoked_certificate_by_serial_number(revoked, serial): - try: - return [x for x in revoked if x.serial_number == serial][0] - except IndexError: - return None - - def _get_extension_for_oid(extensions, oid): - try: - return [x for x in extensions if x.oid == oid][0] - except IndexError: - return None - - changes = {} - - if _getattr_safe(builder, "_issuer_name") != current.issuer: - changes["issuer_name"] = _getattr_safe(builder, "_issuer_name").rfc4514_string() - if not current.is_signature_valid(sig_pubkey): - changes["public_key"] = True - - rev_changes = {"added": [], "changed": [], "removed": []} - revoked = _getattr_safe(builder, "_revoked_certificates") - for rev in revoked: - cur = current.get_revoked_certificate_by_serial_number(rev.serial_number) - if cur is None: - # certificate was not revoked before - rev_changes["added"].append(x509util.dec2hex(rev.serial_number)) - continue - - for ext in rev.extensions: - cur_ext = _get_extension_for_oid(cur.extensions, ext.oid) - # revoked certificate's extensions have changed (added/changed) - if any( - ( - cur_ext is None, - cur_ext.critical != ext.critical, - cur_ext.value != ext.value, - ) - ): - rev_changes["changed"].append(x509util.dec2hex(rev.serial_number)) - - for cur_ext in cur.extensions: - if _get_extension_for_oid(rev.extensions, cur_ext.oid) is None: - # an extension was removed from from the revoked certificate - rev_changes["changed"].append(x509util.dec2hex(rev.serial_number)) - - for rev in current: - # certificate was removed from the CRL, probably because it was outdated anyways - if ( - _get_revoked_certificate_by_serial_number(revoked, rev.serial_number) - is None - ): - rev_changes["removed"].append(x509util.dec2hex(rev.serial_number)) - - if any(rev_changes.values()): - changes["revocations"] = rev_changes - - ext_changes = _compare_exts(current, builder) - if any(ext_changes.values()): - changes["extensions"] = ext_changes - return changes - - -def _compare_exts(current, builder): - def getextname(ext): - try: - return ext.oid._name - except AttributeError: - return ext.oid.dotted_string - - added = [] - changed = [] - removed = [] - builder_extensions = cx509.Extensions(_getattr_safe(builder, "_extensions")) - - # iter is unnecessary, but avoids a pylint < 2.13.6 crash - for ext in iter(builder_extensions): - try: - cur_ext = current.extensions.get_extension_for_oid(ext.value.oid) - if cur_ext.critical != ext.critical or cur_ext.value != ext.value: - changed.append(getextname(ext)) - except cx509.ExtensionNotFound: - added.append(getextname(ext)) - - for ext in current.extensions: - try: - builder_extensions.get_extension_for_oid(ext.value.oid) - except cx509.ExtensionNotFound: - removed.append(getextname(ext)) - - return {"added": added, "changed": changed, "removed": removed} - - -def _compare_ca_chain(current, new): - if not len(current) == len(new): - return False - for i, new_cert in enumerate(new): - if new_cert.fingerprint(hashes.SHA256()) != current[i].fingerprint( - hashes.SHA256() - ): - return False - return True - - -def _getattr_safe(obj, attr): - try: - return getattr(obj, attr) - except AttributeError as err: - # Since we cannot get the certificate object without signing, - # we need to compare attributes marked as internal. At least - # convert possible exceptions into some description. - raise CommandExecutionError( - f"Could not get attribute {attr} from {obj.__class__.__name__}. " - "Did the internal API of cryptography change?" - ) from err - - -def _compareattr_safe(obj, attr, comp): - try: - return getattr(obj, attr) == comp - except AttributeError: - return False - - -def _safe_atomic_write(dst, data, backup): - """ - Create a temporary file with only user r/w perms and atomically - copy it to the destination, honoring ``backup``. - """ - tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX) - with salt.utils.files.fopen(tmp, "wb") as tmp_: - tmp_.write(data) - salt.utils.files.copyfile( - tmp, dst, __salt__["config.backup_mode"](backup), __opts__["cachedir"] - ) - salt.utils.files.safe_rm(tmp) diff --git a/setup/so-functions b/setup/so-functions index b97d4bb52..82dd9d375 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1960,11 +1960,8 @@ salt_install_module_deps() { } salt_patch_x509_v2() { - # this can be removed when https://github.com/saltstack/salt/issues/64195 is resolved - if [ $SALTVERSION == "3006.1" ]; then - info "Salt version 3006.1 found. Patching /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py" - \cp -v ./files/patch/states/x509_v2.py /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py - fi + # this can be removed when https://github.com/saltstack/salt/issues/66929 is resolved + logCmd "salt-call state.apply salt.patch.x509_v2 --local --file-root=../salt/" } # Create an secrets pillar so that passwords survive re-install diff --git a/sigs/securityonion-2.4.150-20250512.iso.sig b/sigs/securityonion-2.4.150-20250512.iso.sig new file mode 100644 index 000000000..7121c68c6 Binary files /dev/null and b/sigs/securityonion-2.4.150-20250512.iso.sig differ