From 0517099e87d34127a5158bf5adf690f4ecc35259 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:54:14 -0600 Subject: [PATCH] remove usage of deprecated 'logs' integration in favor of 'filestream' --- .../grid-nodes_general/import-zeek-logs.json | 22 ++++++++++--- .../grid-nodes_general/kratos-logs.json | 31 ++++++++++++----- .../grid-nodes_general/zeek-logs.json | 24 ++++++++++---- .../grid-nodes_general/hydra-logs.json | 33 +++++++++++++++---- .../grid-nodes_general/idh-logs.json | 28 ++++++++++++---- .../grid-nodes_general/import-evtx-logs.json | 29 +++++++++++----- .../import-suricata-logs.json | 29 ++++++++++++---- .../grid-nodes_general/rita-logs.json | 32 ++++++++++++------ .../grid-nodes_general/so-ip-mappings.json | 28 +++++++++++----- .../soc-auth-sync-logs.json | 28 ++++++++++++---- .../soc-detections-logs.json | 31 ++++++++++++----- .../soc-salt-relay-logs.json | 30 +++++++++++++---- .../soc-sensoroni-logs.json | 30 ++++++++++++----- .../grid-nodes_general/soc-server-logs.json | 30 +++++++++++++---- .../grid-nodes_general/strelka-logs.json | 30 ++++++++++++----- .../grid-nodes_general/suricata-logs.json | 30 ++++++++++++----- 16 files changed, 343 insertions(+), 122 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index 492db03dc..ccb312996 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -2,7 +2,7 @@ {%- raw -%} { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "import-zeek-logs", @@ -10,19 +10,31 @@ "description": "Zeek Import logs", "policy_id": "so-grid-nodes_general", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/zeek/logs/*.log" ], "data_stream.dataset": "import", - "tags": [], + "pipeline": "", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}).log$"], + "include_files": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n" + "tags": [], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json index f6b01cdff..c0214a990 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -11,36 +11,51 @@ {%- endif -%} { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "kratos-logs", - "namespace": "so", "description": "Kratos logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", - "tags": ["so-kratos"], + "pipeline": "kratos", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], {%- if valid_identities -%} "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos\n- if:\n has_fields:\n - identity_id\n then:{% for id, email in identities %}\n - if:\n equals:\n identity_id: \"{{ id }}\"\n then:\n - add_fields:\n target: ''\n fields:\n user.name: \"{{ email }}\"{% endfor %}", {%- else -%} "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", {%- endif -%} - "custom": "pipeline: kratos" + "tags": [ + "so-kratos" + ], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} - +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 5462dc861..062091cef 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -2,28 +2,38 @@ {%- raw -%} { "package": { - "name": "log", + "name": "filestream", "version": "" }, - "id": "zeek-logs", "name": "zeek-logs", "namespace": "so", "description": "Zeek logs", "policy_id": "so-grid-nodes_general", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/*.log" ], "data_stream.dataset": "zeek", - "tags": [], + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}).log$"], + "include_files": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n" + "tags": [], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } @@ -31,4 +41,4 @@ }, "force": true } -{%- endraw -%} +{%- endraw -%} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json index f1b1dace9..9c2c0363b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json @@ -1,26 +1,43 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "hydra-logs", - "namespace": "so", "description": "Hydra logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/opt/so/log/hydra/hydra.log" ], "data_stream.dataset": "hydra", - "tags": ["so-hydra"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: hydra", - "custom": "pipeline: hydra" + "pipeline": "hydra", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: hydra", + "tags": [ + "so-hydra" + ], + "recursive_glob": true, + "ignore_older": "72h", + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } @@ -28,3 +45,5 @@ }, "force": true } + + diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 9f66c1937..002435119 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -1,30 +1,44 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "idh-logs", - "namespace": "so", "description": "IDH integration", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/idh/opencanary.log" ], "data_stream.dataset": "idh", - "tags": [], + "pipeline": "common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- convert:\n fields:\n - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n- drop_fields:\n when:\n equals:\n event.code: \"1001\"\n fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n ignore_missing: true\n- rename:\n fields:\n - from: \"src_host\"\n to: \"source.ip\"\n - from: \"src_port\"\n to: \"source.port\"\n - from: \"dst_host\"\n to: \"destination.host\"\n - from: \"dst_port\"\n to: \"destination.port\"\n ignore_missing: true\n- drop_fields:\n fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n target: event\n fields:\n category: host\n module: opencanary", - "custom": "pipeline: common" + "tags": [], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index dd95e6337..cf6b02d34 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -1,33 +1,46 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "import-evtx-logs", - "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes_general", - "vars": {}, + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/*.json" ], "data_stream.dataset": "import", - "custom": "", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.6.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.6.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.6.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" - ] + ], + "recursive_glob": true, + "ignore_older": "72h", + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index c9b036e36..603a742c8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -1,30 +1,45 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "import-suricata-logs", - "namespace": "so", "description": "Import Suricata logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/suricata/eve*.json" ], "data_stream.dataset": "import", + "pipeline": "suricata.common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], + "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n", "tags": [], - "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", - "custom": "pipeline: suricata.common" + "recursive_glob": true, + "ignore_older": "72h", + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json index a97faaa5f..5e12652ae 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json @@ -1,18 +1,17 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "rita-logs", - "namespace": "so", "description": "RITA Logs", "policy_id": "so-grid-nodes_general", - "vars": {}, + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ @@ -20,15 +19,28 @@ "/nsm/rita/exploded-dns.csv", "/nsm/rita/long-connections.csv" ], - "exclude_files": [], - "ignore_older": "72h", "data_stream.dataset": "rita", - "tags": [], + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- dissect:\n tokenizer: \"/nsm/rita/%{pipeline}.csv\"\n field: \"log.file.path\"\n trim_chars: \".csv\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\").split(\"-\");\n if (pl.length > 1) {\n pl = pl[1];\n }\n else {\n pl = pl[0];\n }\n event.Put(\"@metadata.pipeline\", \"rita.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: rita", - "custom": "exclude_lines: ['^Score', '^Source', '^Domain', '^No results']" + "tags": [], + "recursive_glob": true, + "ignore_older": "72h", + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json index fdcd36815..d8e09e1ae 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json @@ -1,29 +1,41 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "so-ip-mappings", - "namespace": "so", "description": "IP Description mappings", "policy_id": "so-grid-nodes_general", - "vars": {}, + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/custom-mappings/ip-descriptions.csv" ], "data_stream.dataset": "hostnamemappings", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], + "processors": "- decode_csv_fields:\n fields:\n message: decoded.csv\n separator: \",\"\n ignore_missing: false\n overwrite_keys: true\n trim_leading_space: true\n fail_on_error: true\n\n- extract_array:\n field: decoded.csv\n mappings:\n so.ip_address: '0'\n so.description: '1'\n\n- script:\n lang: javascript\n source: >\n function process(event) {\n var ip = event.Get('so.ip_address');\n var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n if (!validIpRegex.test(ip)) {\n event.Cancel();\n }\n }\n- fingerprint:\n fields: [\"so.ip_address\"]\n target_field: \"@metadata._id\"\n", "tags": [ "so-ip-mappings" ], - "processors": "- decode_csv_fields:\n fields:\n message: decoded.csv\n separator: \",\"\n ignore_missing: false\n overwrite_keys: true\n trim_leading_space: true\n fail_on_error: true\n\n- extract_array:\n field: decoded.csv\n mappings:\n so.ip_address: '0'\n so.description: '1'\n\n- script:\n lang: javascript\n source: >\n function process(event) {\n var ip = event.Get('so.ip_address');\n var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n if (!validIpRegex.test(ip)) {\n event.Cancel();\n }\n }\n- fingerprint:\n fields: [\"so.ip_address\"]\n target_field: \"@metadata._id\"\n", - "custom": "" + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } @@ -31,5 +43,3 @@ }, "force": true } - - diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index aa39c177b..8b93a751a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -1,30 +1,44 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "soc-auth-sync-logs", - "namespace": "so", "description": "Security Onion - Elastic Auth Sync - Logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sync.log" ], "data_stream.dataset": "soc", - "tags": ["so-soc"], + "pipeline": "common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync", - "custom": "pipeline: common" + "tags": [], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json index 5649b481d..b8236fa15 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json @@ -1,35 +1,48 @@ { - "policy_id": "so-grid-nodes_general", "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "soc-detections-logs", "description": "Security Onion Console - Detections Logs", + "policy_id": "so-grid-nodes_general", "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/detections_runtime-status_sigma.log", "/opt/so/log/soc/detections_runtime-status_yara.log" ], - "exclude_files": [], - "ignore_older": "72h", "data_stream.dataset": "soc", + "pipeline": "common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: detections\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", "tags": [ "so-soc" ], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: detections\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", - "custom": "pipeline: common" + "recursive_glob": true, + "ignore_older": "72h", + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index cc92092e9..9a27a0c83 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -1,30 +1,46 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "soc-salt-relay-logs", - "namespace": "so", "description": "Security Onion - Salt Relay - Logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/salt-relay.log" ], "data_stream.dataset": "soc", - "tags": ["so-soc"], + "pipeline": "common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay", - "custom": "pipeline: common" + "tags": [ + "so-soc" + ], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 61ad057f4..946cd8b76 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -1,30 +1,44 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "soc-sensoroni-logs", - "namespace": "so", "description": "Security Onion - Sensoroni - Logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/opt/so/log/sensoroni/sensoroni.log" ], "data_stream.dataset": "soc", - "tags": [], + "pipeline": "common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true", - "custom": "pipeline: common" + "tags": [], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, - "force": true -} +"force": true +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index a875e4bfc..941f5b424 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -1,30 +1,46 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "soc-server-logs", - "namespace": "so", "description": "Security Onion Console Logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sensoroni-server.log" ], "data_stream.dataset": "soc", - "tags": ["so-soc"], + "pipeline": "common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", - "custom": "pipeline: common" + "tags": [ + "so-soc" + ], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 89e9bbe8e..a7cbcc013 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -1,30 +1,44 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "strelka-logs", - "namespace": "so", - "description": "Strelka logs", + "description": "Strelka Logs", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "strelka", - "tags": [], + "pipeline": "strelka.file", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", - "custom": "pipeline: strelka.file" + "tags": [], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index c3b04fd86..57eafef0d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -1,30 +1,44 @@ { "package": { - "name": "log", + "name": "filestream", "version": "" }, "name": "suricata-logs", - "namespace": "so", "description": "Suricata integration", "policy_id": "so-grid-nodes_general", + "namespace": "so", "inputs": { - "logs-logfile": { + "filestream-filestream": { "enabled": true, "streams": { - "log.logs": { + "filestream.generic": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], - "data_stream.dataset": "suricata", - "tags": [], + "data_stream.dataset": "filestream.generic", + "pipeline": "suricata.common", + "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", + "exclude_files": [ + "\\.gz$" + ], + "include_files": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", - "custom": "pipeline: suricata.common" + "tags": [], + "recursive_glob": true, + "clean_inactive": -1, + "harvester_limit": 0, + "fingerprint": true, + "fingerprint_offset": 0, + "fingerprint_length": "64", + "file_identity_native": false, + "exclude_lines": [], + "include_lines": [] } } } } }, "force": true -} +} \ No newline at end of file