diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 211a3f58d..852592a10 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -241,7 +241,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal" # docker container getting recycled EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1 + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated fi diff --git a/salt/docker/init.sls b/salt/docker/init.sls index 9945a8096..4917cc163 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -18,10 +18,10 @@ dockergroup: dockerheldpackages: pkg.installed: - pkgs: - - containerd.io: 2.2.1-1.el9 - - docker-ce: 3:29.2.1-1.el9 - - docker-ce-cli: 1:29.2.1-1.el9 - - docker-ce-rootless-extras: 29.2.1-1.el9 + - containerd.io: 2.3.6-1.el9 + - docker-ce: 3:29.8.1-1.el9 + - docker-ce-cli: 1:29.8.1-1.el9 + - docker-ce-rootless-extras: 29.8.1-1.el9 - hold: True - update_holds: True diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 723370ba6..2a3f80431 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -29,7 +29,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.22.3\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.22.3\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.22.3\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- add_fields:\n target: event\n fields:\n dataset: windows.forwarded\n module: windows\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: import.evtx\n- if:\n equals:\n winlog.channel: 'Security'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n- if:\n equals:\n winlog.channel: 'Windows PowerShell'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.powershell\n module: windows\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.application\n module: system\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.system\n module: system\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f67a01df9..c8ceab34d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3309,6 +3309,7 @@ elasticsearch: composed_of: - event-mappings - logs-system.security@package + - so-fleet_system.security_caseless-1 - logs-system.security@custom - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 @@ -4175,6 +4176,7 @@ elasticsearch: index_template: composed_of: - logs-windows.forwarded@package + - so-fleet_process_caseless-1 - logs-windows.forwarded@custom - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 @@ -4224,6 +4226,7 @@ elasticsearch: index_template: composed_of: - logs-windows.powershell@package + - so-fleet_process_caseless-1 - logs-windows.powershell@custom - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 @@ -4273,6 +4276,7 @@ elasticsearch: index_template: composed_of: - logs-windows.powershell_operational@package + - so-fleet_process_caseless-1 - logs-windows.powershell_operational@custom - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 @@ -4322,6 +4326,7 @@ elasticsearch: index_template: composed_of: - logs-windows.sysmon_operational@package + - so-fleet_process_caseless-1 - logs-windows.sysmon_operational@custom - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 979c5c1b8..ab1418cd2 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -99,7 +99,7 @@ }, { "set": { - "if": "ctx.tags != null && ctx.tags.contains('import')", + "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')", "override": true, "field": "data_stream.dataset", "value": "import" @@ -107,7 +107,7 @@ }, { "set": { - "if": "ctx.tags != null && ctx.tags.contains('import')", + "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')", "override": true, "field": "data_stream.namespace", "value": "so" diff --git a/salt/elasticsearch/files/ingest/import.evtx b/salt/elasticsearch/files/ingest/import.evtx new file mode 100644 index 000000000..99189130c --- /dev/null +++ b/salt/elasticsearch/files/ingest/import.evtx @@ -0,0 +1,31 @@ +{ + "description" : "import.evtx: normalize imported EVTX and reroute to logs--import", + "processors" : [ + { "script": { + "description": "Host from the event, not the importing node", + "lang": "painless", + "source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;" + } }, + { "script": { + "description": "String event IDs, as Winlogbeat sends", + "lang": "painless", + "source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }" + } }, + { "script": { + "description": "Unnamed to param1..N, as Winlogbeat", + "lang": "painless", + "if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null", + "source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');" + } }, + { "script": { + "description": "String values and LF line endings, as Winlogbeat", + "lang": "painless", + "if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map", + "source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }" + } }, + { "set": { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } }, + { "set": { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } }, + { "set": { "field": "data_stream.namespace", "value": "import", "override": true } }, + { "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } } + ] +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json new file mode 100644 index 000000000..37b6413e9 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json @@ -0,0 +1,123 @@ +{ + "_meta": { + "managed_by": "security_onion", + "managed": true, + "description": "Adds .caseless for Lucene queries. Restates each field's package type and .text." + }, + "template": { + "mappings": { + "properties": { + "process": { + "properties": { + "executable": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + } + }, + "command_line": { + "type": "wildcard", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + } + }, + "parent": { + "properties": { + "executable": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + } + }, + "command_line": { + "type": "wildcard", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + } + } + } + } + } + }, + "file": { + "properties": { + "path": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + } + } + } + } + } + } + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json new file mode 100644 index 000000000..d320364fb --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json @@ -0,0 +1,80 @@ +{ + "_meta": { + "managed_by": "security_onion", + "managed": true, + "description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type." + }, + "template": { + "mappings": { + "properties": { + "process": { + "properties": { + "command_line": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + } + } + }, + "parent": { + "properties": { + "executable": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + } + } + }, + "command_line": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + } + } + } + } + } + } + }, + "file": { + "properties": { + "path": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + } + } + } + } + } + } + } + } +} diff --git a/salt/repo/client/map.jinja b/salt/repo/client/map.jinja index 21f52a5e7..3a44d8431 100644 --- a/salt/repo/client/map.jinja +++ b/salt/repo/client/map.jinja @@ -9,7 +9,7 @@ 'epel-testing.repo', 'saltstack.repo', 'salt-latest.repo', - 'wazuh.repo' + 'wazuh.repo', 'Rocky-Base.repo', 'Rocky-CR.repo', 'Rocky-Debuginfo.repo', diff --git a/salt/soc/files/soc/sigma_pipelines/sigma_so_pipeline.yml b/salt/soc/files/soc/sigma_pipelines/sigma_so_pipeline.yml index fd89b56bd..89b24f53b 100644 --- a/salt/soc/files/soc/sigma_pipelines/sigma_so_pipeline.yml +++ b/salt/soc/files/soc/sigma_pipelines/sigma_so_pipeline.yml @@ -14,6 +14,16 @@ transformations: - process.args - related.ip - dns.resolved_ip + # Not every source maps .caseless; EQL/ES|QL already match case-insensitively. + - id: caseless_to_parent_fields + type: field_name_mapping + mapping: + process.executable.caseless: process.executable + process.name.caseless: process.name + process.parent.executable.caseless: process.parent.executable + process.parent.name.caseless: process.parent.name + target.process.executable.caseless: target.process.executable + target.process.name.caseless: target.process.name - id: baseline_field_name_mapping type: field_name_mapping mapping: diff --git a/salt/soc/files/soc/sigma_playbook_pipeline.yaml b/salt/soc/files/soc/sigma_playbook_pipeline.yaml index a779c4dec..b31f8ba5d 100644 --- a/salt/soc/files/soc/sigma_playbook_pipeline.yaml +++ b/salt/soc/files/soc/sigma_playbook_pipeline.yaml @@ -2,13 +2,13 @@ name: Security Onion - Playbook Pipeline priority: 97 transformations: # Route to lowercase-normalized .caseless subfields for case-insensitive matching. - # file.path.caseless exists on Defend only (Sysmon file events lack it); # registry.path / dll.path / file.name have no .caseless on any source. - id: case_insensitive_string_fields type: field_name_mapping mapping: process.executable: process.executable.caseless process.parent.executable: process.parent.executable.caseless + process.parent.name: process.parent.name.caseless process.command_line: process.command_line.caseless process.parent.command_line: process.parent.command_line.caseless file.path: file.path.caseless