Merge pull request #15733 from Security-Onion-Solutions/reyesj2-es932

filestream integration policy updates
This commit is contained in:
Jorge Reyes
2026-04-07 11:14:10 -05:00
committed by GitHub
14 changed files with 140 additions and 59 deletions
@@ -6,21 +6,23 @@
"name": "agent-monitor", "name": "agent-monitor",
"namespace": "", "namespace": "",
"description": "", "description": "",
"policy_id": "so-grid-nodes_general",
"policy_ids": [ "policy_ids": [
"so-grid-nodes_general" "so-grid-nodes_general"
], ],
"output_id": null,
"vars": {}, "vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/opt/so/log/agents/agent-monitor.log" "/opt/so/log/agents/agent-monitor.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "agentmonitor", "data_stream.dataset": "agentmonitor",
"pipeline": "elasticagent.monitor", "pipeline": "elasticagent.monitor",
"parsers": "", "parsers": "",
@@ -34,15 +36,16 @@
"ignore_older": "72h", "ignore_older": "72h",
"clean_inactive": -1, "clean_inactive": -1,
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": true, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": 64, "file_identity_native": true,
"file_identity_native": false,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
} }
} },
"force": true
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "hydra-logs", "name": "hydra-logs",
"namespace": "so",
"description": "Hydra logs", "description": "Hydra logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/opt/so/log/hydra/hydra.log" "/opt/so/log/hydra/hydra.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "hydra", "data_stream.dataset": "hydra",
"pipeline": "hydra", "pipeline": "hydra",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -34,10 +40,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "idh-logs", "name": "idh-logs",
"namespace": "so",
"description": "IDH integration", "description": "IDH integration",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/nsm/idh/opencanary.log" "/nsm/idh/opencanary.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "idh", "data_stream.dataset": "idh",
"pipeline": "common", "pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "import-evtx-logs", "name": "import-evtx-logs",
"namespace": "so",
"description": "Import Windows EVTX logs", "description": "Import Windows EVTX logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/nsm/import/*/evtx/*.json" "/nsm/import/*/evtx/*.json"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "import", "data_stream.dataset": "import",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [ "exclude_files": [
@@ -33,10 +39,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "import-suricata-logs", "name": "import-suricata-logs",
"namespace": "so",
"description": "Import Suricata logs", "description": "Import Suricata logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/nsm/import/*/suricata/eve*.json" "/nsm/import/*/suricata/eve*.json"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "import", "data_stream.dataset": "import",
"pipeline": "suricata.common", "pipeline": "suricata.common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -32,10 +38,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,14 +4,18 @@
"version": "" "version": ""
}, },
"name": "rita-logs", "name": "rita-logs",
"namespace": "so",
"description": "RITA Logs", "description": "RITA Logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
@@ -19,6 +23,8 @@
"/nsm/rita/exploded-dns.csv", "/nsm/rita/exploded-dns.csv",
"/nsm/rita/long-connections.csv" "/nsm/rita/long-connections.csv"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "rita", "data_stream.dataset": "rita",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [ "exclude_files": [
@@ -33,10 +39,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "so-ip-mappings", "name": "so-ip-mappings",
"namespace": "so",
"description": "IP Description mappings", "description": "IP Description mappings",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/nsm/custom-mappings/ip-descriptions.csv" "/nsm/custom-mappings/ip-descriptions.csv"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "hostnamemappings", "data_stream.dataset": "hostnamemappings",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [ "exclude_files": [
@@ -32,10 +38,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "soc-auth-sync-logs", "name": "soc-auth-sync-logs",
"namespace": "so",
"description": "Security Onion - Elastic Auth Sync - Logs", "description": "Security Onion - Elastic Auth Sync - Logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/opt/so/log/soc/sync.log" "/opt/so/log/soc/sync.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc", "data_stream.dataset": "soc",
"pipeline": "common", "pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,20 +4,26 @@
"version": "" "version": ""
}, },
"name": "soc-detections-logs", "name": "soc-detections-logs",
"namespace": "so",
"description": "Security Onion Console - Detections Logs", "description": "Security Onion Console - Detections Logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/opt/so/log/soc/detections_runtime-status_sigma.log", "/opt/so/log/soc/detections_runtime-status_sigma.log",
"/opt/so/log/soc/detections_runtime-status_yara.log" "/opt/so/log/soc/detections_runtime-status_yara.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc", "data_stream.dataset": "soc",
"pipeline": "common", "pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -35,10 +41,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "soc-salt-relay-logs", "name": "soc-salt-relay-logs",
"namespace": "so",
"description": "Security Onion - Salt Relay - Logs", "description": "Security Onion - Salt Relay - Logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/opt/so/log/soc/salt-relay.log" "/opt/so/log/soc/salt-relay.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc", "data_stream.dataset": "soc",
"pipeline": "common", "pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -33,10 +39,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "soc-sensoroni-logs", "name": "soc-sensoroni-logs",
"namespace": "so",
"description": "Security Onion - Sensoroni - Logs", "description": "Security Onion - Sensoroni - Logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/opt/so/log/sensoroni/sensoroni.log" "/opt/so/log/sensoroni/sensoroni.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc", "data_stream.dataset": "soc",
"pipeline": "common", "pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "soc-server-logs", "name": "soc-server-logs",
"namespace": "so",
"description": "Security Onion Console Logs", "description": "Security Onion Console Logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/opt/so/log/soc/sensoroni-server.log" "/opt/so/log/soc/sensoroni-server.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc", "data_stream.dataset": "soc",
"pipeline": "common", "pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -33,10 +39,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "strelka-logs", "name": "strelka-logs",
"namespace": "so",
"description": "Strelka Logs", "description": "Strelka Logs",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/nsm/strelka/log/strelka.log" "/nsm/strelka/log/strelka.log"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "strelka", "data_stream.dataset": "strelka",
"pipeline": "strelka.file", "pipeline": "strelka.file",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }
@@ -4,19 +4,25 @@
"version": "" "version": ""
}, },
"name": "suricata-logs", "name": "suricata-logs",
"namespace": "so",
"description": "Suricata integration", "description": "Suricata integration",
"policy_id": "so-grid-nodes_general", "policy_id": "so-grid-nodes_general",
"namespace": "so", "policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": { "inputs": {
"filestream-filestream": { "filestream-filestream": {
"enabled": true, "enabled": true,
"streams": { "streams": {
"filestream.generic": { "filestream.filestream": {
"enabled": true, "enabled": true,
"vars": { "vars": {
"paths": [ "paths": [
"/nsm/suricata/eve*.json" "/nsm/suricata/eve*.json"
], ],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "suricata", "data_stream.dataset": "suricata",
"pipeline": "suricata.common", "pipeline": "suricata.common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0, "harvester_limit": 0,
"fingerprint": false, "fingerprint": false,
"fingerprint_offset": 0, "fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true, "file_identity_native": true,
"exclude_lines": [], "exclude_lines": [],
"include_lines": [] "include_lines": [],
"delete_enabled": false
} }
} }
} }