Compare commits

..
17 changed files with 244 additions and 76 deletions
@@ -5,7 +5,7 @@
"package": {
"name": "endpoint",
"title": "Elastic Defend",
"version": "9.3.1",
"version": "9.4.1",
"requires_root": true
},
"enabled": true,
@@ -29,7 +29,7 @@
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.22.3\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.22.3\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.22.3\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"tags": [
"import"
],
@@ -16,7 +16,6 @@
'awsfirehose.metrics': 'aws.cloudwatch',
'cribl.logs': 'cribl',
'cribl.metrics': 'cribl',
'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
'azure_application_insights.app_insights': 'azure.app_insights',
'azure_application_insights.app_state': 'azure.app_state',
'azure_billing.billing': 'azure.billing',
+7
View File
@@ -98,6 +98,13 @@ so-es-cluster-settings:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
- http: wait_for_so-elasticsearch
so-elasticsearch-system-indices-patch:
cmd.run:
- name: /usr/sbin/so-elasticsearch-system-indices-patch
- require:
- http: wait_for_so-elasticsearch
- file: so-elasticsearch-system-indices-patch-script
{% endif %}
# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
+10
View File
@@ -42,6 +42,16 @@ elasticsearch_sbin:
- file_mode: 755
- exclude_pat:
- so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
- so-elasticsearch-system-indices-patch
- show_changes: False
so-elasticsearch-system-indices-patch-script:
file.managed:
- name: /usr/sbin/so-elasticsearch-system-indices-patch
- source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
- user: 930
- group: 939
- mode: 755
- show_changes: False
elasticsearch_sbin_jinja:
+1 -1
View File
@@ -1,7 +1,7 @@
elasticsearch:
enabled: false
esheap: '600m'
version: 9.3.7
version: 9.4.5
index_clean: true
data_retention_method: DLM
vm:
@@ -0,0 +1,199 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
set -eo pipefail
SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
KIBANA_PASSWORD=
INDEX_PATTERNS=(
'.entity_analytics.risk_score.lookup-*'
'.entity_analytics.watchlists.*'
'.entity_analytics.monitoring.users-*'
'.entity_analytics.entity-leads-*'
'.asset-criticality.asset-criticality-*'
'.workflows-executions'
'.workflows-step-executions'
'.entities.v2.latest.security_*'
'.entities.v2.history.security_*'
'risk-score.risk-score-latest-*'
)
DATA_STREAM_PATTERNS=(
'.entities.v2.updates.security_*'
'risk-score.risk-score-*'
'.rule-events'
'.alert-actions'
)
TEMPLATE_PATTERNS=(
'entities_v2_latest_security_default_index_template'
'entities_v2_history_security_default_index_template'
'.entities_v2_updates_security_default_index_template'
'.risk-score.risk-score-default-index-template'
'.rule-events'
'.alert-actions'
)
query_es() {
if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
return 0
fi
# retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
local query_path="$1"
shift
if [[ -z "$KIBANA_PASSWORD" ]]; then
KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
fi
[[ -n "$KIBANA_PASSWORD" ]] || return 1
echo "Retrying ${query_path} as so_kibana." >&2
curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
-s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
}
# add auto_expand_replicas=0-1 to given index
set_auto_expand_replicas() {
local index="$1"
echo "Setting auto_expand_replicas to 0-1 on ${index}."
query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
}
# resolve index patterns and find each index with an unassigned replica
unassigned_replicas() {
local pattern="$1"
local resolved_indices response index
if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
return 0
fi
while read -r index; do
if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
continue
fi
jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
}
data_stream_indices() {
local pattern="$1"
local response
if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
return 0
fi
jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
}
update_system_indices() {
local pattern="$1"
local index
while read -r index; do
[[ -n "$index" ]] && set_auto_expand_replicas "$index"
done < <(unassigned_replicas "$pattern")
}
# update data stream backing indices with unassigned replicas
update_system_ds() {
local pattern="$1"
local index
while read -r index; do
while read -r unassigned_index; do
[[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
done < <(unassigned_replicas "$index")
done < <(data_stream_indices "$pattern")
}
has_unassigned_replicas() {
local pattern="$1"
local index
index=$(unassigned_replicas "$pattern" | sed -n '1p')
[[ -n "$index" ]]
}
data_stream_has_unassigned_replicas() {
local pattern="$1"
local index
while read -r index; do
has_unassigned_replicas "$index" && return 0
done < <(data_stream_indices "$pattern")
return 1
}
needs_patch() {
local pattern
for pattern in "${INDEX_PATTERNS[@]}"; do
has_unassigned_replicas "$pattern" && return 0
done
for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
data_stream_has_unassigned_replicas "$pattern" && return 0
done
return 1
}
# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
update_system_templates() {
local pattern="$1"
local templates name response template auto_expand_replicas
if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
return 0
fi
while read -r name; do
response=$(query_es "_index_template/${name}")
template=$(jq -c '.index_templates[0].index_template' <<<"$response")
auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
[[ "$auto_expand_replicas" == "0-1" ]] && continue
template=$(jq '
if (.template.settings.index | type) == "object" then
.template.settings.index.auto_expand_replicas = "0-1"
else
.template.settings["index.auto_expand_replicas"] = "0-1"
end
| del(.created_date_millis, .modified_date_millis)
' <<<"$template")
echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
done < <(jq -r '.index_templates[]?.name' <<<"$templates")
}
if [[ "${1:-}" == "--check" ]]; then
needs_patch
exit $?
fi
if [[ $# -ne 0 ]]; then
echo "Usage: $0 [--check]" >&2
exit 1
fi
patched=false
for pattern in "${INDEX_PATTERNS[@]}"; do
if has_unassigned_replicas "$pattern"; then
update_system_indices "$pattern"
patched=true
fi
done
for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
if data_stream_has_unassigned_replicas "$pattern"; then
update_system_ds "$pattern"
patched=true
fi
done
if [[ "$patched" == true ]]; then
for pattern in "${TEMPLATE_PATTERNS[@]}"; do
update_system_templates "$pattern"
done
fi
+1 -1
View File
@@ -22,7 +22,7 @@ kibana:
- default
- file
migrations:
discardCorruptObjects: "9.3.7"
discardCorruptObjects: "9.4.5"
telemetry:
enabled: False
xpack:
@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
# Disable certain Features from showing up in the Kibana UI
echo
echo "Setting up default Kibana Space:"
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
echo
+1 -23
View File
@@ -344,16 +344,6 @@ check_cluster_health() {
check_fleet_server() {
echo "Checking that Elastic Fleet Server is responding."
# Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks
local protected_symlinks=$(sysctl -b fs.protected_symlinks)
if [[ "$protected_symlinks" == "1" ]]; then
# disable fs.protected_symlinks and restart elasticfleet
sysctl -w fs.protected_symlinks=0
docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
salt-call state.apply elasticfleet queue=True
fi
# Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
# which waits for HTTP 200 from the Fleet Server status API.
if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
@@ -1046,20 +1036,8 @@ post_to_3.2.0() {
}
### 3.2.0 End ###
### 3.3.0 Scripts ###
# Sets fs.protected_symlinks=0
#
# Elastic Agent docker image chowns its directory to the running UID
# but does not chown the elastic-agent launcher symlink.
# Preventing non-root users from following that launcher symlink.
disable_sysctl_fs_protected_symlink() {
salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true
}
### 3.2.0 Scripts ###
up_to_3.3.0() {
disable_sysctl_fs_protected_symlink
INSTALLEDVERSION=3.3.0
}
+2 -20
View File
@@ -3,8 +3,6 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'vars/globals.map.jinja' import GLOBALS %}
# Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
# RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
# per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -21,25 +19,9 @@ so_boot_highstate_unit_file:
- onchanges_in:
- module: systemd_reload
# Non-managers never apply salt.minion during setup, so reaching this state means
# setup is finished and the marker is safe to write unconditionally. This also
# heals nodes installed before this fix, which have no marker and no legacy
# startup_states line to grep for. Managers do highstate mid-setup, so they only
# get the marker from the legacy upgrade signal; fresh installs get it from
# mark_setup_complete in setup/so-functions.
mark_setup_complete:
file.managed:
- name: /opt/so/state/setup-complete
- replace: false
- makedirs: True
{% if GLOBALS.is_manager %}
- onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
{% endif %}
- require_in:
- service: so_boot_highstate_service
# Only enable once setup is complete. Until then the gate file is missing and
# the unit's own ConditionPathExists would no-op it anyway.
# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
# `systemctl is-enabled` honest for the sync_es_users gate.
so_boot_highstate_service:
service.enabled:
- name: so-boot-highstate.service
+16 -4
View File
@@ -87,15 +87,27 @@ set_log_levels:
# so-boot-highstate.service (managed in salt.minion.boot_highstate), which
# runs once per system boot only. Strip the line from /etc/salt/minion on
# upgrade; both the commented and uncommented forms historically existed.
# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager
# gate there greps for this line, so it must run before we delete it.
remove_startup_states:
file.line:
- name: /etc/salt/minion
- match: 'startup_states: highstate'
- mode: delete
- require:
- file: mark_setup_complete
# Upgrade-path bridge: systems that already passed setup under the old gate
# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
# the marker from `mark_setup_complete` in setup/so-functions at the right
# moment. `replace: false` means we never overwrite a marker once written.
mark_setup_complete_for_upgrades:
file.managed:
- name: /opt/so/state/setup-complete
- replace: false
- makedirs: True
- onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
- require_in:
- file: remove_startup_states
- service: so_boot_highstate_service
{% endif %}
-9
View File
@@ -8,15 +8,6 @@ set_role_grain:
- name: role
- value: so-{{ grains.id.split("_") | last }}
# salt-cloud guests never run so-setup, so nothing else marks them setup-complete.
# Replaces the 'startup_states: highstate' line this state used to append. No
# GLOBALS import -- this runs before the guest's pillars exist.
mark_setup_complete_vm_guest:
file.managed:
- name: /opt/so/state/setup-complete
- replace: false
- makedirs: True
enable_salt_minion:
service.enabled:
- name: salt-minion
+2 -2
View File
@@ -2671,7 +2671,7 @@ soc:
# The id (UUIDv4) is pregenerated and can safely be used.
# Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
#
# Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
# Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
# Logsources: https://sigmahq.io/docs/basics/log-sources.html
title: 'A Short Capitalized Title With Less Than 50 Characters'
@@ -2683,7 +2683,7 @@ soc:
references:
- 'https://local.invalid'
author: '@SecurityOnion'
date: '[today]'
date: 'YYYY/MM/DD'
tags:
- detection.threat_hunting
- attack.technique_id
-9
View File
@@ -65,15 +65,6 @@ run_remediate:
- success_retcodes:
- 2
# Elastic Agent docker image chowns its directory to the running UID but does not
# chown the elastic-agent launcher symlink. fs.protected_symlinks=1 then prevents
# non-root users from following that launcher symlink.
{# OSCAP rule id: xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks #}
fs.protected_symlinks:
sysctl.present:
- value: 0
- config: /etc/sysctl.conf
{# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #}
disable_ctrl_alt_del_action:
file.replace:
+2 -2
View File
@@ -1601,7 +1601,7 @@ DISA STIG for Oracle Linux 9 V1R3.</xccdf-1.2:description>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
@@ -2202,7 +2202,7 @@ standard DISA STIG for Oracle Linux 9 profile.</xccdf-1.2:description>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
-1
View File
@@ -833,7 +833,6 @@ if ! [[ -f $install_opt_file ]]; then
check_sos_appliance
drop_install_options
hypervisor_local_states
mark_setup_complete
verify_setup
fi