CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/2.4/dev' into vlb2

Browse Source
This commit is contained in:
Josh Patterson
2025-04-23 15:42:04 -04:00
parent 285d73d526 69c904548c
commit ed80c4e13b
17 changed files with 256 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
salt/elasticfleet/defaults.yaml
View File

@@ -11,6 +11,7 @@ elasticfleet:
defend_filters:
enable_auto_configuration: False
subscription_integrations: False
auto_upgrade_integrations: False
logging:
zeek:
excluded:

46
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json Normal file
View File

@@ -0,0 +1,46 @@
{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%}
{%- set valid_identities = false -%}
{%- if identities -%}
{%- set valid_identities = true -%}
{%- for id, email in identities -%}
{%- if not id or not email -%}
{%- set valid_identities = false -%}
{%- break -%}
{%- endif -%}
{%- endfor -%}
{%- endif -%}
{
"package": {
"name": "log",
"version": ""
},
"name": "kratos-logs",
"namespace": "so",
"description": "Kratos logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,
"streams": {
"log.logs": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/kratos/kratos.log"
],
"data_stream.dataset": "kratos",
"tags": ["so-kratos"],
{%- if valid_identities -%}
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos\n- if:\n has_fields:\n - identity_id\n then:{% for id, email in identities %}\n - if:\n equals:\n identity_id: \"{{ id }}\"\n then:\n - add_fields:\n target: ''\n fields:\n user.name: \"{{ email }}\"{% endfor %}",
{%- else -%}
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos",
{%- endif -%}
"custom": "pipeline: kratos"
}
}
}
}
},
"force": true
}

30
salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json
View File

@@ -1,30 +0,0 @@
{
"package": {
"name": "log",
"version": ""
},
"name": "kratos-logs",
"namespace": "so",
"description": "Kratos logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,
"streams": {
"log.logs": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/kratos/kratos.log"
],
"data_stream.dataset": "kratos",
"tags": ["so-kratos"],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos",
"custom": "pipeline: kratos"
}
}
}
}
},
"force": true
}

5
salt/elasticfleet/soc_elasticfleet.yaml
View File

@@ -45,6 +45,11 @@ elasticfleet:
global: True
forcedType: bool
helpLink: elastic-fleet.html
auto_upgrade_integrations:
description: Enables or disables automatically upgrading Elastic Agent integrations.
global: True
forcedType: bool
helpLink: elastic-fleet.html
server:
custom_fqdn:
description: Custom FQDN for Agents to connect to. One per line.

62
salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade
View File

@@ -1,62 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-elastic-fleet-common
curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
if [ $? -ne 0 ]; then
echo "Error: Failed to connect to Kibana."
exit 1
fi
IFS=$'\n'
agent_policies=$(elastic_fleet_agent_policy_ids)
if [ $? -ne 0 ]; then
echo "Error: Failed to retrieve agent policies."
exit 1
fi
for AGENT_POLICY in $agent_policies; do
integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
for INTEGRATION in $integrations; do
if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
# Get package name so we know what package to look for when checking the current and latest available version
PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
# Get currently installed version of package
PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
# Get latest available version of package
AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
# Get integration ID
INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
# Dry run of the upgrade
echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
echo "Upgrading $INTEGRATION..."
echo "Starting dry run..."
DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
# If no errors with dry run, proceed with actual upgrade
if [[ "$DRYRUN_ERRORS" == "false" ]]; then
echo "No errors detected. Proceeding with upgrade..."
elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
if [ $? -ne 0 ]; then
echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'."
exit 1
fi
else
echo "Errors detected during dry run. Stopping upgrade..."
exit 1
fi
fi
fi
done
done
echo

73
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade Normal file
View File

@@ -0,0 +1,73 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
. /usr/sbin/so-elastic-fleet-common
curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
if [ $? -ne 0 ]; then
echo "Error: Failed to connect to Kibana."
exit 1
fi
IFS=$'\n'
agent_policies=$(elastic_fleet_agent_policy_ids)
if [ $? -ne 0 ]; then
echo "Error: Failed to retrieve agent policies."
exit 1
fi
default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
for AGENT_POLICY in $agent_policies; do
integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
for INTEGRATION in $integrations; do
if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
# Get package name so we know what package to look for when checking the current and latest available version
PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
{%- endif %}
# Get currently installed version of package
PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
# Get latest available version of package
AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
# Get integration ID
INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
# Dry run of the upgrade
echo ""
echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
echo "Upgrading $INTEGRATION..."
echo "Starting dry run..."
DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
# If no errors with dry run, proceed with actual upgrade
if [[ "$DRYRUN_ERRORS" == "false" ]]; then
echo "No errors detected. Proceeding with upgrade..."
elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
if [ $? -ne 0 ]; then
echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
exit 1
fi
else
echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
exit 1
fi
fi
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
fi
{%- endif %}
fi
done
done
echo

55
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
View File

@@ -3,7 +3,10 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
# this file except in compliance with the Elastic License 2.0.
{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %}
{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
. /usr/sbin/so-common
. /usr/sbin/so-elastic-fleet-common
@@ -46,6 +49,28 @@ compare_versions() {
fi
}
IFS=$'\n'
agent_policies=$(elastic_fleet_agent_policy_ids)
if [ $? -ne 0 ]; then
echo "Error: Failed to retrieve agent policies."
exit 1
fi
default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
in_use_integrations=()
for AGENT_POLICY in $agent_policies; do
integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
for INTEGRATION in $integrations; do
PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
# non-default integrations that are in-use in any policy
if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
in_use_integrations+=("$PACKAGE_NAME")
fi
done
done
if [[ -f $STATE_FILE_SUCCESS ]]; then
if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then
# Package_list contains all integrations beta / non-beta.
@@ -77,10 +102,19 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
else
results=$(compare_versions "$latest_version" "$installed_version")
if [ $results == "greater" ]; then
echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
{#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #}
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
{%- endif %}
echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
PENDING_UPDATE=true
PENDING_UPDATE=true
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
else
echo "skipping available upgrade for in use integration - $package_name."
fi
{%- endif %}
fi
fi
fi
@@ -92,9 +126,18 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
else
results=$(compare_versions "$latest_version" "$installed_version")
if [ $results == "greater" ]; then
echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
PENDING_UPDATE=true
{#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #}
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
{%- endif %}
echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
PENDING_UPDATE=true
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
else
echo "skipping available upgrade for in use integration - $package_name."
fi
{%- endif %}
fi
fi
{% endif %}

2
salt/elasticsearch/auth.sls
View File

@@ -15,7 +15,7 @@
elastic_auth_pillar:
file.managed:
- name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls
- mode: 600
- mode: 640
- reload_pillar: True
- contents: |
elasticsearch:

7
salt/elasticsearch/enabled.sls
View File

@@ -204,12 +204,17 @@ so-elasticsearch-roles-load:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% if GLOBALS.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager', 'so-managerhype'] %}
{% if grains.role in ['so-managersearch', 'so-heavynode', 'so-manager', 'so-managerhype'] %}
{% set ap = "absent" %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone'] %}
{% if ELASTICSEARCHMERGED.index_clean %}
{% set ap = "present" %}
{% else %}
{% set ap = "absent" %}
{% endif %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
so-elasticsearch-indices-delete:
cron.{{ap}}:
- name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1

4
salt/kafka/nodes.sls
View File

@@ -10,9 +10,9 @@
write_kafka_pillar_yaml:
file.managed:
- name: /opt/so/saltstack/local/pillar/kafka/nodes.sls
- mode: 644
- mode: 640
- user: socore
- source: salt://kafka/files/managed_node_pillar.jinja
- template: jinja
- context:
COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }}
COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }}

2
salt/kibana/secrets.sls
View File

@@ -22,7 +22,7 @@ kibana_pillar_directory:
kibana_secrets_pillar:
file.managed:
- name: /opt/so/saltstack/local/pillar/kibana/secrets.sls
- mode: 600
- mode: 640
- reload_pillar: True
- contents: |
kibana:

10
salt/kratos/config.sls
View File

@@ -13,13 +13,21 @@ kratosgroup:
- name: kratos
- gid: 928
kratoshome:
file.directory:
- name: /opt/so/conf/kratos
- user: 928
- group: 928
- mode: 700
- makedirs: True
# Add Kratos user
kratos:
user.present:
- uid: 928
- gid: 928
- home: /opt/so/conf/kratos
kratosdir:
file.directory:
- name: /nsm/kratos

10
salt/logstash/config.sls
View File

@@ -22,7 +22,15 @@ logstashgroup:
- name: logstash
- gid: 931
# Add the logstash user for the jog4j settings
logstashhome:
file.directory:
- name: /opt/so/conf/logstash
- user: 931
- group: 931
- mode: 700
- makedirs: True
# Add the logstash user for the log4j settings
logstash:
user.present:
- uid: 931

2
salt/manager/elasticsearch.sls
View File

@@ -3,5 +3,5 @@ elastic_curl_config_distributed:
- name: /opt/so/saltstack/local/salt/elasticsearch/curl.config
- source: salt://elasticsearch/files/curl.config.template
- template: jinja
- mode: 600
- mode: 640
- show_changes: False

17
salt/manager/init.sls
View File

@@ -127,15 +127,28 @@ so_fleetagent_status:
- month: '*'
- dayweek: '*'
socore_own_saltstack:
socore_own_saltstack_default:
file.directory:
- name: /opt/so/saltstack
- name: /opt/so/saltstack/default
- user: socore
- group: socore
- recurse:
- user
- group
socore_own_saltstack_local:
file.directory:
- name: /opt/so/saltstack/local
- user: socore
- group: socore
- dir_mode: 750
- file_mode: 640
- replace: False
- recurse:
- user
- group
- mode
rules_dir:
file.directory:
- name: /nsm/rules/yara

2
salt/manager/tools/sbin/so-minion
View File

@@ -200,7 +200,7 @@ function testMinion() {
function restartMinion() {
log "INFO" "Restarting minion $MINION_ID"
salt "$MINION_ID" system.reboot
salt "$MINION_ID" system.reboot --async
result=$?
if [ $result -ne 0 ]; then

37
salt/soc/soc_soc.yaml
View File

@@ -138,6 +138,43 @@ soc:
title: Require TOTP
description: Require all users to enable Time-based One Time Passwords (MFA) upon login to SOC.
global: True
subgrids:
title: Subordinate Grids
description: |
Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must be granted most permissions in order to perform required functions.
*Requires a valid Security Onion license key with subgrid allocations.*
global: True
syntax: json
forcedType: "[]{}"
uiElements:
- field: id
label: Unique Subgrid ID
regex: "^((?!_)).+$"
regexFailureMessage: Subgrid ID cannot start with an underscore
required: true
- field: managerUrl
label: Subgrid Manager URL
required: true
- field: clientId
label: Subgrid API Client ID
required: true
regex: "^socl_[a-z0-9_]+$"
regexFailureMessage: Client ID must be a valid socl_* API Client ID
- field: clientSecret
label: Subgrid API Client Secret
required: true
- field: tlsSkipVerify
label: Skip Subgrid TLS Certification Validation
forcedType: bool
default: false
- field: caCertificate
label: Subgrid CA Certificate
multiline: True
- field: enabled
label: Subgrid Enabled
forcedType: bool
default: false
modules:
elastalertengine:
aiRepoUrl: