Compare commits

..
Author SHA1 Message Date
Jorge ReyesandGitHub 58672a0743 Remove wait_for_so-elastic-fleet state
Removed wait_for_so-elastic-fleet state from enabled.sls.
2026-07-14 16:27:07 -05:00
Jason ErtelandGitHub 1657480d31 Merge pull request #16064 from Security-Onion-Solutions/jertel/wip
prevent login redirect to any API url
2026-07-14 13:55:44 -04:00
Jason Ertel 63d4061500 prevent login redirect to any API url 2026-07-14 13:52:32 -04:00
Jorge ReyesandGitHub 8167ae3282 Merge pull request #16062 from Security-Onion-Solutions/reyesj2/es937
ES 9.3.7
2026-07-13 14:35:35 -05:00
reyesj2 2cd889782d soup es check for 9.3.7 2026-07-13 13:16:12 -05:00
reyesj2 87a5639643 pipeline updates 2026-07-13 13:05:25 -05:00
reyesj2 99e9fc1c3b ES 9.3.7 2026-07-10 13:50:55 -05:00
13 changed files with 29 additions and 369 deletions
-1
View File
@@ -1,6 +1,5 @@
elasticfleet:
enabled: False
patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
enable_manager_output: True
config:
server:
-11
View File
@@ -111,17 +111,6 @@ so-elastic-fleet:
- file: eastatedir
- x509: etc_elasticfleet_key
- x509: etc_elasticfleet_crt
wait_for_so-elastic-fleet:
http.wait_for_successful_query:
- name: "https://localhost:8220/api/status"
- ssl: True
- verify_ssl: False
- status: 200
- wait_for: 300
- request_interval: 15
- require:
- docker_container: so-elastic-fleet
{% endif %}
delete_so-elastic-fleet_so-status.disabled:
@@ -5,7 +5,7 @@
"package": {
"name": "endpoint",
"title": "Elastic Defend",
"version": "9.3.0",
"version": "9.3.1",
"requires_root": true
},
"enabled": true,
@@ -29,7 +29,7 @@
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"tags": [
"import"
],
+1 -1
View File
@@ -1,6 +1,6 @@
elasticsearch:
enabled: false
version: 9.3.3
version: 9.3.7
index_clean: true
data_retention_method: DLM
vm:
@@ -118,70 +118,70 @@
{
"pipeline": {
"tag": "pipeline_e16851a7",
"name": "logs-pfsense.log-1.25.2-firewall",
"name": "logs-pfsense.log-1.25.4-firewall",
"if": "ctx.event.provider == 'filterlog'"
}
},
{
"pipeline": {
"tag": "pipeline_828590b5",
"name": "logs-pfsense.log-1.25.2-openvpn",
"name": "logs-pfsense.log-1.25.4-openvpn",
"if": "ctx.event.provider == 'openvpn'"
}
},
{
"pipeline": {
"tag": "pipeline_9d37039c",
"name": "logs-pfsense.log-1.25.2-ipsec",
"name": "logs-pfsense.log-1.25.4-ipsec",
"if": "ctx.event.provider == 'charon'"
}
},
{
"pipeline": {
"tag": "pipeline_ad56bbca",
"name": "logs-pfsense.log-1.25.2-dhcp",
"name": "logs-pfsense.log-1.25.4-dhcp",
"if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
}
},
{
"pipeline": {
"tag": "pipeline_dd85553d",
"name": "logs-pfsense.log-1.25.2-unbound",
"name": "logs-pfsense.log-1.25.4-unbound",
"if": "ctx.event.provider == 'unbound'"
}
},
{
"pipeline": {
"tag": "pipeline_720ed255",
"name": "logs-pfsense.log-1.25.2-haproxy",
"name": "logs-pfsense.log-1.25.4-haproxy",
"if": "ctx.event.provider == 'haproxy'"
}
},
{
"pipeline": {
"tag": "pipeline_456beba5",
"name": "logs-pfsense.log-1.25.2-php-fpm",
"name": "logs-pfsense.log-1.25.4-php-fpm",
"if": "ctx.event.provider == 'php-fpm'"
}
},
{
"pipeline": {
"tag": "pipeline_a0d89375",
"name": "logs-pfsense.log-1.25.2-squid",
"name": "logs-pfsense.log-1.25.4-squid",
"if": "ctx.event.provider == 'squid'"
}
},
{
"pipeline": {
"tag": "pipeline_c2f1ed55",
"name": "logs-pfsense.log-1.25.2-snort",
"name": "logs-pfsense.log-1.25.4-snort",
"if": "ctx.event.provider == 'snort'"
}
},
{
"pipeline": {
"tag":"pipeline_33db1c9e",
"name": "logs-pfsense.log-1.25.2-suricata",
"name": "logs-pfsense.log-1.25.4-suricata",
"if": "ctx.event.provider == 'suricata'"
}
},
+1 -1
View File
@@ -22,7 +22,7 @@ kibana:
- default
- file
migrations:
discardCorruptObjects: "9.3.3"
discardCorruptObjects: "9.3.7"
telemetry:
enabled: False
xpack:
+6 -3
View File
@@ -740,7 +740,6 @@ fix_logstash_0013_lumberjack_pipeline_name() {
up_to_3.1.0() {
ensure_postgres_local_pillar
ensure_postgres_secret
determine_elastic_agent_upgrade
elasticsearch_backup_index_templates
# Clear existing component template state file.
rm -f /opt/so/state/esfleet_component_templates.json
@@ -888,6 +887,9 @@ update_kafka_metadata() {
}
up_to_3.2.0() {
# download 9.3.7 elastic agent packages
determine_elastic_agent_upgrade
fix_logstash_0013_lumberjack_pipeline_name
pin_elasticsearch_data_retention_method
@@ -901,7 +903,7 @@ post_to_3.2.0() {
bootstrap_so_soc_database
# Including agent regen script here since it was missed in post_to_3.1.0
# Generate 9.3.7 elastic agent installers
echo "Regenerating Elastic Agent Installers"
/sbin/so-elastic-agent-gen-installers
@@ -1174,7 +1176,8 @@ verify_es_version_compatibility() {
["8.18.4"]="8.18.6 8.18.8 9.0.8"
["8.18.6"]="8.18.8 9.0.8"
["8.18.8"]="9.0.8"
["9.0.8"]="9.3.3"
["9.0.8"]="9.3.3 9.3.7"
["9.3.3"]="9.3.7"
)
# Elasticsearch MUST upgrade through these versions
+1 -1
View File
@@ -399,7 +399,7 @@ http {
error_page 429 = @error429;
location @error401 {
if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
return 401;
}
@@ -14,22 +14,18 @@
CommandLine: process.command_line
CurrentDirectory: process.working_directory
Details: registry.data.strings
Image: process.executable
ImageLoaded: dll.path
ImageLoaded: dll.name
ParentImage: process.parent.executable
ParentName: process.parent.name
ParentProcessGuid: process.parent.entity_id
ProcessGuid: process.entity_id
TargetFilename: file.path
TargetFilename: file.name
TargetObject: registry.path
TargetUserName: user.target.name
User: user.name
community_id: network.community_id
dns_resolved_ip: dns.resolved_ip
QueryName: dns.question.name
dll_basename: dll.name
file_basename: file.name
document_id: soc_id
dst_ip: destination.ip
dst_port: destination.port
@@ -44,15 +40,10 @@ public_ip: network.public_ip
related_hosts: related.hosts
related_ip: related.ip
src_ip: source.ip
dns_query_name: dns.question.name
dns_query_name: dns.query_name
flow_id: log.id.uid
payload: network.data.decoded
rule_category: rule.category
rule_name: rule.name
rule_uuid: rule.uuid
src_port: source.port
# PowerShell channel (ps_script EID 4104 / classic EID 400)
ScriptBlockText: powershell.file.script_block_text
script_block_id: powershell.file.script_block_id
script_block_hash: powershell.file.script_block_hash
EngineVersion: powershell.engine.version
+2 -121
View File
@@ -1,9 +1,8 @@
name: Security Onion - Playbook Pipeline
priority: 97
transformations:
# Route to lowercase-normalized .caseless subfields for case-insensitive matching.
# file.path.caseless exists on Defend only (Sysmon file events lack it);
# registry.path / dll.path / file.name have no .caseless on any source.
# Route string fields to their lowercase-normalized .caseless subfield so wildcard
# matches are case-insensitive.
- id: case_insensitive_string_fields
type: field_name_mapping
mapping:
@@ -11,121 +10,3 @@ transformations:
process.parent.executable: process.parent.executable.caseless
process.command_line: process.command_line.caseless
process.parent.command_line: process.parent.command_line.caseless
file.path: file.path.caseless
# file_activity: playbook-only pseudo-category spanning all file operations.
- id: playbook_file_activity_add-fields
type: add_condition
conditions:
event.category: 'file'
rule_conditions:
- type: logsource
category: file_activity
# EventType -> event.action is safe as a projection only — the values differ
# (Sysmon DeleteFile/SetValue vs ECS deletion/modification).
- id: playbook_file_activity_event_type
type: field_name_mapping
mapping:
winlog.event_data.EventType: event.action
rule_conditions:
- type: logsource
category: file_activity
- id: playbook_file_event_event_type
type: field_name_mapping
mapping:
winlog.event_data.EventType: event.action
rule_conditions:
- type: logsource
category: file_event
- id: playbook_file_delete_event_type
type: field_name_mapping
mapping:
winlog.event_data.EventType: event.action
rule_conditions:
- type: logsource
category: file_delete
- id: playbook_file_rename_event_type
type: field_name_mapping
mapping:
winlog.event_data.EventType: event.action
rule_conditions:
- type: logsource
category: file_rename
- id: playbook_registry_set_event_type
type: field_name_mapping
mapping:
winlog.event_data.EventType: event.action
rule_conditions:
- type: logsource
category: registry_set
# WMI-Activity events carry no event.category; event.dataset is source-specific
- id: playbook_wmi_activity_add-fields
type: add_condition
conditions:
event.dataset: 'wmi.operational'
rule_conditions:
- type: logsource
category: wmi
- id: playbook_wmi_event_activity_add-fields
type: add_condition
conditions:
event.dataset: 'wmi.operational'
rule_conditions:
- type: logsource
category: wmi_event
- id: playbook_ps_script_field-mapping
type: field_name_mapping
mapping:
ScriptBlockText: powershell.file.script_block_text
Path: file.path
rule_conditions:
- type: logsource
category: ps_script
- id: playbook_ps_script_add-fields
type: add_condition
conditions:
event.dataset: 'windows.powershell_operational'
event.code: '4104'
rule_conditions:
- type: logsource
category: ps_script
# Data (raw EID 400/600 message blob) -> process.command_line: the winlog
# integration parses HostApplication into it; `message` is analyzed text and
# wildcard matches on it silently return zero. No .caseless on this dataset.
- id: playbook_ps_classic_start_field-mapping
type: field_name_mapping
mapping:
Data: process.command_line
rule_conditions:
- type: logsource
category: ps_classic_start
- id: playbook_ps_classic_start_add-fields
type: add_condition
conditions:
event.dataset: 'windows.powershell'
event.code: '400'
rule_conditions:
- type: logsource
category: ps_classic_start
- id: playbook_ps_classic_provider_start_field-mapping
type: field_name_mapping
mapping:
Data: process.command_line
rule_conditions:
- type: logsource
category: ps_classic_provider_start
- id: playbook_ps_classic_provider_start_add-fields
type: add_condition
conditions:
event.dataset: 'windows.powershell'
event.code: '600'
rule_conditions:
- type: logsource
category: ps_classic_provider_start
# Alert docs nest the host under event_data.host.name; no top-level host.name.
- id: playbook_alert_host_field
type: field_name_mapping
mapping:
host.name: event_data.host.name
rule_conditions:
- type: logsource
category: alert
+3 -206
View File
@@ -131,22 +131,13 @@ transformations:
type: field_name_mapping
mapping:
winlog.event_data.Details: registry.data.strings
# field rename only; EventType values (SetValue/CreateKey) still differ from
# event.action values (modification/creation)
winlog.event_data.EventType: event.action
rule_conditions:
- type: logsource
product: windows
category: registry_set
# ecs_windows renames Initiated -> network.direction without translating the value,
# so rules compile to network.direction:"true" and never match (field holds
# ingress/egress). network.initiated carries the boolean on both Defend and Sysmon.
# Keyed on network.direction because ecs_windows has already renamed by this layer.
- id: ecs_fix_network_connection_initiated
type: field_name_mapping
mapping:
network.direction: network.initiated
rule_conditions:
- type: logsource
product: windows
category: network_connection
- id: ecs_fix_image_load
type: field_name_mapping
mapping:
@@ -159,30 +150,6 @@ transformations:
- type: logsource
product: windows
category: image_load
# Defend reports the driver image in dll.path (file.path is null on driver events) —
# same renames as image_load.
- id: ecs_fix_driver_load
type: field_name_mapping
mapping:
file.path: dll.path
file.code_signature.signed: dll.code_signature.exists
winlog.event_data.Signature: dll.code_signature.subject_name
file.code_signature.status: dll.code_signature.status
winlog.event_data.Hashes: dll.hash.sha256
rule_conditions:
- type: logsource
product: windows
category: driver_load
- id: ecs_fix_file_rename
type: field_name_mapping
mapping:
# Defend records the pre-rename path in file.Ext.original.path; Sysmon's
# SourceFilename has no ECS equivalent.
winlog.event_data.SourceFilename: file.Ext.original.path
rule_conditions:
- type: logsource
product: windows
category: file_rename
- id: linux_security_add-fields
type: add_condition
conditions:
@@ -356,33 +323,6 @@ transformations:
rule_conditions:
- type: logsource
category: file_event
# Without event.type scoping, file_delete rules also match creations.
- id: endpoint_file_delete_add-fields
type: add_condition
conditions:
event.category: 'file'
event.type: 'deletion'
rule_conditions:
- type: logsource
category: file_delete
# Defend reports renames as event.action:rename with event.type:change.
- id: endpoint_file_rename_add-fields
type: add_condition
conditions:
event.category: 'file'
event.action: 'rename'
rule_conditions:
- type: logsource
category: file_rename
# event.type:change selects writes and excludes Defend's read-only registry events.
- id: endpoint_registry_set_add-fields
type: add_condition
conditions:
event.category: 'registry'
event.type: 'change'
rule_conditions:
- type: logsource
category: registry_set
# Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
# populated).
- id: endpoint_image_load_add-fields
@@ -411,149 +351,6 @@ transformations:
rule_conditions:
- type: logsource
category: network_connection
# Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53).
- id: endpoint_dns_query_add-fields
type: add_condition
conditions:
event.category: 'network'
event.action:
- 'lookup_requested'
- 'lookup_result'
rule_conditions:
- type: logsource
category: dns_query
# OS gates: without them a `product: windows` rule in these categories also matches
# Linux/macOS. Separate conditions (not `product:` on the mappings above) so the
# mappings keep scoping product-less rules.
- id: endpoint_file_create_windows_os-gate
type: add_condition
conditions:
host.os.type: 'windows'
rule_conditions:
- type: logsource
category: file_event
product: windows
- id: endpoint_file_create_linux_os-gate
type: add_condition
conditions:
host.os.type: 'linux'
rule_conditions:
- type: logsource
category: file_event
product: linux
- id: endpoint_file_create_macos_os-gate
type: add_condition
conditions:
host.os.type: 'macos'
rule_conditions:
- type: logsource
category: file_event
product: macos
- id: endpoint_file_delete_windows_os-gate
type: add_condition
conditions:
host.os.type: 'windows'
rule_conditions:
- type: logsource
category: file_delete
product: windows
- id: endpoint_file_delete_linux_os-gate
type: add_condition
conditions:
host.os.type: 'linux'
rule_conditions:
- type: logsource
category: file_delete
product: linux
- id: endpoint_file_delete_macos_os-gate
type: add_condition
conditions:
host.os.type: 'macos'
rule_conditions:
- type: logsource
category: file_delete
product: macos
- id: endpoint_file_rename_windows_os-gate
type: add_condition
conditions:
host.os.type: 'windows'
rule_conditions:
- type: logsource
category: file_rename
product: windows
- id: endpoint_file_rename_linux_os-gate
type: add_condition
conditions:
host.os.type: 'linux'
rule_conditions:
- type: logsource
category: file_rename
product: linux
- id: endpoint_file_rename_macos_os-gate
type: add_condition
conditions:
host.os.type: 'macos'
rule_conditions:
- type: logsource
category: file_rename
product: macos
- id: endpoint_network_connection_windows_os-gate
type: add_condition
conditions:
host.os.type: 'windows'
rule_conditions:
- type: logsource
category: network_connection
product: windows
- id: endpoint_network_connection_linux_os-gate
type: add_condition
conditions:
host.os.type: 'linux'
rule_conditions:
- type: logsource
category: network_connection
product: linux
- id: endpoint_network_connection_macos_os-gate
type: add_condition
conditions:
host.os.type: 'macos'
rule_conditions:
- type: logsource
category: network_connection
product: macos
- id: endpoint_dns_query_windows_os-gate
type: add_condition
conditions:
host.os.type: 'windows'
rule_conditions:
- type: logsource
category: dns_query
product: windows
- id: endpoint_dns_query_linux_os-gate
type: add_condition
conditions:
host.os.type: 'linux'
rule_conditions:
- type: logsource
category: dns_query
product: linux
- id: endpoint_dns_query_macos_os-gate
type: add_condition
conditions:
host.os.type: 'macos'
rule_conditions:
- type: logsource
category: dns_query
product: macos
# registry_set / image_load / driver_load are not gated — windows-only telemetry.
# wmi / wmi_event are not scoped here: their only handle is event.dataset
- id: endpoint_driver_load_add-fields
type: add_condition
conditions:
event.category: 'driver'
rule_conditions:
- type: logsource
category: driver_load
# Maps "alert" category to SO Alert events
- id: alert_so_add-fields
type: add_condition