mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 17:22:49 +01:00
Merge pull request #15234 from Security-Onion-Solutions/reyesj2/pipeline-upd
update zeek pipelines
This commit is contained in:
Jorge Reyes
GitHub
@@ -15,7 +15,6 @@ elasticfleet:
|
||||
logging:
|
||||
zeek:
|
||||
excluded:
|
||||
- analyzer
|
||||
- broker
|
||||
- capture_loss
|
||||
- cluster
|
||||
|
||||
61
salt/elasticsearch/files/ingest/zeek.analyzer
Normal file
61
salt/elasticsearch/files/ingest/zeek.analyzer
Normal file
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"description": "zeek.analyzer",
|
||||
"processors": [
|
||||
{
|
||||
"set": {
|
||||
"field": "event.dataset",
|
||||
"value": "analyzer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": [
|
||||
"host"
|
||||
],
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"field": "message",
|
||||
"target_field": "message2",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"set": {
|
||||
"field": "network.protocol",
|
||||
"copy_from": "message2.analyzer_name",
|
||||
"ignore_empty_value": true,
|
||||
"if": "ctx?.message2?.analyzer_kind == 'protocol'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"set": {
|
||||
"field": "network.protocol",
|
||||
"ignore_empty_value": true,
|
||||
"if": "ctx?.message2?.analyzer_kind != 'protocol'",
|
||||
"copy_from": "message2.proto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"lowercase": {
|
||||
"field": "network.protocol",
|
||||
"ignore_missing": true,
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.failure_reason",
|
||||
"target_field": "error.reason",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "zeek.common"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,35 +1,227 @@
|
||||
{
|
||||
"description" : "zeek.dns",
|
||||
"processors" : [
|
||||
{ "set": { "field": "event.dataset", "value": "dns" } },
|
||||
{ "remove": { "field": ["host"], "ignore_failure": true } },
|
||||
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
|
||||
{ "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } },
|
||||
{ "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.trans_id", "target_field": "dns.id", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.rtt", "target_field": "event.duration", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.query", "target_field": "dns.query.name", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.qclass", "target_field": "dns.query.class", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.qclass_name", "target_field": "dns.query.class_name", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.qtype", "target_field": "dns.query.type", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.qtype_name", "target_field": "dns.query.type_name", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.rcode", "target_field": "dns.response.code", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.rcode_name", "target_field": "dns.response.code_name", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.AA", "target_field": "dns.authoritative", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.TC", "target_field": "dns.truncated", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.RD", "target_field": "dns.recursion.desired", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.RA", "target_field": "dns.recursion.available", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.Z", "target_field": "dns.reserved", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.answers", "target_field": "dns.answers.name", "ignore_missing": true } },
|
||||
{ "foreach": {"field": "dns.answers.name","processor": {"pipeline": {"name": "common.ip_validation"}},"if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null","ignore_failure": true}},
|
||||
{ "foreach": {"field": "temp._valid_ips","processor": {"append": {"field": "dns.resolved_ip","allow_duplicates": false,"value": "{{{_ingest._value}}}","ignore_failure": true}},"ignore_failure": true}},
|
||||
{ "script": { "source": "if (ctx.dns.resolved_ip != null && ctx.dns.resolved_ip instanceof List) {\n ctx.dns.resolved_ip.removeIf(item -> item == null || item.toString().trim().isEmpty());\n }","ignore_failure": true }},
|
||||
{ "remove": {"field": ["temp"], "ignore_missing": true ,"ignore_failure": true } },
|
||||
{ "rename": { "field": "message2.TTLs", "target_field": "dns.ttls", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } },
|
||||
{ "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } },
|
||||
{ "set": { "if": "ctx._index == 'so-zeek'", "field": "_index", "value": "so-zeek_dns", "override": true } },
|
||||
{ "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
|
||||
{ "pipeline": { "name": "zeek.common" } }
|
||||
]
|
||||
"description": "zeek.dns",
|
||||
"processors": [
|
||||
{
|
||||
"set": {
|
||||
"field": "event.dataset",
|
||||
"value": "dns"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": [
|
||||
"host"
|
||||
],
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"field": "message",
|
||||
"target_field": "message2",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"dot_expander": {
|
||||
"field": "id.orig_h",
|
||||
"path": "message2",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.proto",
|
||||
"target_field": "network.transport",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.trans_id",
|
||||
"target_field": "dns.id",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.rtt",
|
||||
"target_field": "event.duration",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.query",
|
||||
"target_field": "dns.query.name",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.qclass",
|
||||
"target_field": "dns.query.class",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.qclass_name",
|
||||
"target_field": "dns.query.class_name",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.qtype",
|
||||
"target_field": "dns.query.type",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.qtype_name",
|
||||
"target_field": "dns.query.type_name",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.rcode",
|
||||
"target_field": "dns.response.code",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.rcode_name",
|
||||
"target_field": "dns.response.code_name",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.AA",
|
||||
"target_field": "dns.authoritative",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.TC",
|
||||
"target_field": "dns.truncated",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.RD",
|
||||
"target_field": "dns.recursion.desired",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.RA",
|
||||
"target_field": "dns.recursion.available",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.Z",
|
||||
"target_field": "dns.reserved",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.answers",
|
||||
"target_field": "dns.answers.name",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"foreach": {
|
||||
"field": "dns.answers.name",
|
||||
"processor": {
|
||||
"pipeline": {
|
||||
"name": "common.ip_validation"
|
||||
}
|
||||
},
|
||||
"if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"foreach": {
|
||||
"field": "temp._valid_ips",
|
||||
"processor": {
|
||||
"append": {
|
||||
"field": "dns.resolved_ip",
|
||||
"allow_duplicates": false,
|
||||
"value": "{{{_ingest._value}}}",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
"if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"script": {
|
||||
"source": "if (ctx.dns.resolved_ip != null && ctx.dns.resolved_ip instanceof List) {\n ctx.dns.resolved_ip.removeIf(item -> item == null || item.toString().trim().isEmpty());\n }",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": [
|
||||
"temp"
|
||||
],
|
||||
"ignore_missing": true,
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.TTLs",
|
||||
"target_field": "dns.ttls",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.rejected",
|
||||
"target_field": "dns.query.rejected",
|
||||
"ignore_missing": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"script": {
|
||||
"lang": "painless",
|
||||
"source": "ctx.dns.query.length = ctx.dns.query.name.length()",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"set": {
|
||||
"if": "ctx._index == 'so-zeek'",
|
||||
"field": "_index",
|
||||
"value": "so-zeek_dns",
|
||||
"override": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')",
|
||||
"name": "dns.tld"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "zeek.common"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"description" : "zeek.dpd",
|
||||
"processors" : [
|
||||
{ "set": { "field": "event.dataset", "value": "dpd" } },
|
||||
{ "remove": { "field": ["host"], "ignore_failure": true } },
|
||||
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
|
||||
{ "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } },
|
||||
{ "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } },
|
||||
{ "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } },
|
||||
{ "rename": { "field": "message2.id.orig_p", "target_field": "source.port", "ignore_missing": true } },
|
||||
{ "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } },
|
||||
{ "rename": { "field": "message2.id.resp_h", "target_field": "destination.ip", "ignore_missing": true } },
|
||||
{ "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } },
|
||||
{ "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.analyzer", "target_field": "observer.analyzer", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.failure_reason", "target_field": "error.reason", "ignore_missing": true } },
|
||||
{ "pipeline": { "name": "zeek.common" } }
|
||||
]
|
||||
}
|
||||
@@ -1746,7 +1746,7 @@ soc:
|
||||
showSubtitle: true
|
||||
- name: DPD
|
||||
description: Dynamic Protocol Detection errors
|
||||
query: 'tags:dpd | groupby error.reason'
|
||||
query: '(tags:dpd OR tags:analyzer) | groupby error.reason'
|
||||
showSubtitle: true
|
||||
- name: Files
|
||||
description: Files grouped by mimetype
|
||||
@@ -2012,7 +2012,7 @@ soc:
|
||||
query: 'tags:dns | groupby dns.query.name | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby dns.highest_registered_domain | groupby dns.parent_domain | groupby dns.query.type_name | groupby dns.response.code_name | groupby dns.answers.name | groupby destination.as.organization.name'
|
||||
- name: DPD
|
||||
description: DPD (Dynamic Protocol Detection) errors
|
||||
query: 'tags:dpd | groupby error.reason | groupby -sankey error.reason source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby destination.as.organization.name'
|
||||
query: '(tags:dpd OR tags:analyzer) | groupby error.reason | groupby -sankey error.reason source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby destination.as.organization.name'
|
||||
- name: Files
|
||||
description: Files seen in network traffic
|
||||
query: 'tags:file | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip | groupby destination.as.organization.name'
|
||||
|
||||
@@ -45,7 +45,7 @@ zeek:
|
||||
- protocols/ssh/geo-data
|
||||
- protocols/ssh/detect-bruteforcing
|
||||
- protocols/ssh/interesting-hostnames
|
||||
- protocols/http/detect-sqli
|
||||
- protocols/http/detect-sql-injection
|
||||
- frameworks/files/hash-all-files
|
||||
- frameworks/files/detect-MHR
|
||||
- policy/frameworks/notice/extend-email/hostnames
|
||||
|
||||
Reference in New Issue
Block a user