CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-09 18:52:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

modify pipelines

Browse Source
This commit is contained in:
Wes Lambert
2020-03-14 12:03:32 +00:00
parent d48c2723ba
commit c52220330b
12 changed files with 103 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
salt/elasticsearch/files/ingest/common
View File

@@ -1,17 +1,10 @@
{
"description" : "common",
"processors" : [
{
"rename": {
"field": "type",
"target_field": "event_type",
"ignore_missing": true
}
},
{
"geoip": {
"field": "destination.ip",
"target_field": "destination_geo",
"target_field": "geo",
"database_file": "GeoLite2-City.mmdb",
"ignore_missing": true,
"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
@@ -20,7 +13,7 @@
{
"geoip": {
"field": "source.ip",
"target_field": "source_geo",
"target_field": "geo",
"database_file": "GeoLite2-City.mmdb",
"ignore_missing": true,
"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
@@ -41,10 +34,12 @@
"ignore_failure": true,
"index_name_format": "yyyy.MM.dd"
}
},
},
{ "rename": { "field": "module", "target_field": "event.module", "ignore_missing": true } },
{ "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_missing": true } },
{
"remove": {
"field": [ "index_name_prefix"],
"field": [ "index_name_prefix", "beat"],
"ignore_failure": false
}
}

53
salt/elasticsearch/files/ingest/ossec.alert Normal file
View File

@@ -0,0 +1,53 @@
{
"description" : "ossec",
"processors" : [
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.agent", "target_field": "agent", "ignore_missing": true } },
{ "rename": { "field": "message2.data", "target_field": "data", "ignore_missing": true } },
{ "rename": { "field": "message2.decoder", "target_field": "decoder", "ignore_missing": true } },
{ "rename": { "field": "message2.full_log", "target_field": "log.full", "ignore_missing": true } },
{ "rename": { "field": "message2.id", "target_field": "log.id.id", "ignore_missing": true } },
{ "rename": { "field": "message2.location", "target_field": "location", "ignore_missing": true } },
{ "rename": { "field": "message2.manager", "target_field": "manager", "ignore_missing": true } },
{ "rename": { "field": "message2.predecoder", "target_field": "predecoder", "ignore_missing": true } },
{ "rename": { "field": "message2.timestamp", "target_field": "timestamp", "ignore_missing": true } },
{ "rename": { "field": "message2.previous_log", "target_field": "log.previous_log", "ignore_missing": true } },
{ "rename": { "field": "message2.previous_output", "target_field": "log.previous_output", "ignore_missing": true } },
{ "rename": { "field": "message2.rule", "target_field": "rule", "ignore_missing": true } },
{ "rename": { "field": "data.command", "target_field": "command", "ignore_missing": true } },
{ "rename": { "field": "data.dstip", "target_field": "destination.ip", "ignore_missing": true } },
{ "rename": { "field": "data.dstport", "target_field": "destination.port", "ignore_missing": true } },
{ "rename": { "field": "data.dstuser", "target_field": "user.escalated", "ignore_missing": true } },
{ "rename": { "field": "data.srcip", "target_field": "source.ip", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.destinationHostname", "target_field": "destination.hostname", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.destinationIp", "target_field": "destination.ip", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.destinationPort", "target_field": "destination.port", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.image", "target_field": "image_path", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.parentImage", "target_field": "parent_image_path", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.sourceHostname", "target_field": "source.hostname", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.sourceIp", "target_field": "source_ip", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.sourcePort", "target_field": "source.port", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.targetFilename", "target_field": "file.target", "ignore_missing": true } },
{ "rename": { "field": "data.win.eventdata.user", "target_field": "user.name", "ignore_missing": true } },
{ "rename": { "field": "data.win.system.eventID", "target_field": "event.code", "ignore_missing": true } },
{ "rename": { "field": "predecoder.program_name", "target_field": "process.name", "ignore_missing": true } },
{ "set": { "if": "ctx.rule.level == 1", "field": "rule.category", "value": "None" } },
{ "set": { "if": "ctx.rule.level == 2", "field": "rule.category", "value": "System low priority notification" } },
{ "set": { "if": "ctx.rule.level == 3", "field": "rule.category", "value": "Successful/authorized event" } },
{ "set": { "if": "ctx.rule.level == 4", "field": "rule.category", "value": "System low priority error" } },
{ "set": { "if": "ctx.rule.level == 5", "field": "rule.category", "value": "User generated error" } },
{ "set": { "if": "ctx.rule.level == 6", "field": "rule.category", "value": "Low relevance attack" } },
{ "set": { "if": "ctx.rule.level == 7", "field": "rule.category", "value": "\"Bad word\" matching" } },
{ "set": { "if": "ctx.rule.level == 8", "field": "rule.category", "value": "First time seen" } },
{ "set": { "if": "ctx.rule.level == 9", "field": "rule.category", "value": "Error from invalid source" } },
{ "set": { "if": "ctx.rule.level == 10", "field": "rule.category", "value": "Multiple user generated errors" } },
{ "set": { "if": "ctx.rule.level == 11", "field": "rule.category", "value": "Integrity checking warning" } },
{ "set": { "if": "ctx.rule.level == 12", "field": "rule.category", "value": "High importance event" } },
{ "set": { "if": "ctx.rule.level == 13", "field": "rule.category", "value": "Unusal error (high importance)" } },
{ "set": { "if": "ctx.rule.level == 14", "field": "rule.category", "value": "High importance security event" } },
{ "set": { "if": "ctx.rule.level == 15", "field": "rule.category", "value": "Severe attack" } },
{ "append": { "if": "ctx.rule.level != null", "field": "tags", "value": ["alert"] } },
{ "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } },
{ "pipeline": { "name": "common" } }
]
}

12
salt/elasticsearch/files/ingest/strelka.file Normal file
View File

@@ -0,0 +1,12 @@
{
"description" : "strelka",
"processors" : [
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.file", "target_field": "file", "ignore_missing": true } },
{ "rename": { "field": "message2.scan", "target_field": "scan", "ignore_missing": true } },
{ "rename": { "field": "message2.request", "target_field": "request", "ignore_missing": true } },
{ "rename": { "field": "scan.hash", "target_field": "file.hash", "ignore_missing": true } },
{ "remove": { "field": ["host", "path"], "ignore_missing": true } },
{ "pipeline": { "name": "common" } }
]
}

11
salt/elasticsearch/files/ingest/suricata.alert Normal file
View File

@@ -0,0 +1,11 @@
{
"description" : "suricata.alert",
"processors" : [
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "rename":{ "field": "message2.comunity_id", "target_field": "network.comunity_id", "ignore_failure": true } },
{ "rename":{ "field": "message2.alert", "target_field": "rule", "ignore_failure": true } },
{ "rename":{ "field": "rule.signature", "target_field": "rule.name", "ignore_failure": true } },
{ "rename":{ "field": "rule.ref", "target_field": "rule.version", "ignore_failure": true } },
{ "pipeline": { "name": "suricata.common" } }
]
}

14
salt/elasticsearch/files/ingest/suricata.common Normal file
View File

@@ -0,0 +1,14 @@
{
"description" : "suricata.common",
"processors" : [
{ "rename":{ "field": "message2.proto", "target_field": "network.transport", "ignore_failure": true } },
{ "rename":{ "field": "message2.flow_id", "target_field": "event.id", "ignore_failure": true } },
{ "rename":{ "field": "message2.comunity_id", "target_field": "network.comunity_id", "ignore_failure": true } },
{ "rename":{ "field": "message2.src_ip", "target_field": "source.ip", "ignore_failure": true } },
{ "rename":{ "field": "message2.src_port", "target_field": "source.port", "ignore_failure": true } },
{ "rename":{ "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } },
{ "rename":{ "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } },
{ "remove": { "field": ["message2"], "ignore_failure": true } },
{ "pipeline": { "name": "common" } }
]
}

4
salt/elasticsearch/files/ingest/zeek.common
View File

@@ -1,7 +1,7 @@
{
"description" : "zeek.common",
"processors" : [
{ "rename": { "field": "@timestamp", "target_field": "es.timestamp", "ignore_missing": true } },
{ "rename": { "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } },
{ "rename": { "field": "message2.uid", "target_field": "log.id.uid", "ignore_missing": true } },
{ "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } },
@@ -15,8 +15,6 @@
{ "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } },
{ "set": { "field": "server.port", "value": "{{destination.port}}" } },
{ "set": { "field": "event.module", "value": "zeek" } },
{ "grok": { "field": "event_type", "patterns": ["zeek.%{WORD:event.dataset}"] } },
{ "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } },
{ "remove": { "field": ["message2.ts", "path"], "ignore_failure": true } },
{ "pipeline": { "name": "common" } }

16
salt/elasticsearch/files/ingest/zeek.conn
View File

@@ -1,7 +1,7 @@
{
"description" : "zeek.conn",
"processors" : [
{ "remove": { "field": ["host"], "ignore_failure": false } },
{ "remove": { "field": ["host"], "ignore_failure": true } },
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } },
{ "rename": { "field": "message2.service", "target_field": "network.protocol", "ignore_missing": true } },
@@ -21,20 +21,6 @@
{ "rename": { "field": "message2.orig_cc", "target_field": "client.country_code","ignore_missing": true } },
{ "rename": { "field": "message2.resp_cc", "target_field": "server.country_code", "ignore_missing": true } },
{ "rename": { "field": "message2.sensorname", "target_field": "observer.name", "ignore_missing": true } },
{ "script": { "lang": "painless", "source": "ctx.connection.bytes.total = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } },
{ "set": { "if": "ctx.connection.state == 'S0'", "field": "connection.state_description", "value": "Connection attempt seen, no reply" } },
{ "set": { "if": "ctx.connection.state == 'S1'", "field": "connection.state_description", "value": "Connection established, not terminated" } },
{ "set": { "if": "ctx.connection.state == 'S2'", "field": "connection.state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } },
{ "set": { "if": "ctx.connection.state == 'S3'", "field": "connection.state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } },
{ "set": { "if": "ctx.connection.state == 'SF'", "field": "connection.state_description", "value": "Normal SYN/FIN completion" } },
{ "set": { "if": "ctx.connection.state == 'REJ'", "field": "connection.state_description", "value": "Connection attempt rejected" } },
{ "set": { "if": "ctx.connection.state == 'RSTO'", "field": "connection.state_description", "value": "Connection established, originator aborted (sent a RST)" } },
{ "set": { "if": "ctx.connection.state == 'RSTR'", "field": "connection.state_description", "value": "Established, responder aborted" } },
{ "set": { "if": "ctx.connection.state == 'RSTOS0'","field": "connection.state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } },
{ "set": { "if": "ctx.connection.state == 'RSTRH'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } },
{ "set": { "if": "ctx.connection.state == 'SH'", "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
{ "set": { "if": "ctx.connection.state == 'SHR'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
{ "set": { "if": "ctx.connection.state == 'OTH'", "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
{ "pipeline": { "name": "zeek.common" } }
]
}

5
salt/elasticsearch/files/ingest/zeek.dpd
View File

@@ -3,7 +3,6 @@
"processors" : [
{ "remove": { "field": ["host"], "ignore_failure": true } },
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } },
{ "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } },
{ "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } },
@@ -13,8 +12,8 @@
{ "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } },
{ "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } },
{ "rename": { "field": "message2.analyzer", "target_field": "analyzer", "ignore_missing": true } },
{ "rename": { "field": "message2.failure_reason", "target_field": "failure_reason", "ignore_missing": true } },
{ "rename": { "field": "message2.analyzer", "target_field": "observer.analyzer", "ignore_missing": true } },
{ "rename": { "field": "message2.failure_reason", "target_field": "error.reason", "ignore_missing": true } },
{ "pipeline": { "name": "zeek.common" } }
]
}

2
salt/elasticsearch/files/ingest/zeek.http
View File

@@ -2,7 +2,7 @@
"description" : "zeek.http",
"processors" : [
{ "remove": { "field": ["host"], "ignore_failure": true } },
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.trans_depth", "target_field": "http.trans_depth", "ignore_missing": true } },
{ "rename": { "field": "message2.method", "target_field": "http.method", "ignore_missing": true } },
{ "rename": { "field": "message2.host", "target_field": "http.virtual_host", "ignore_missing": true } },

1
salt/elasticsearch/files/ingest/zeek.tunnel
View File

@@ -2,7 +2,6 @@
"description" : "zeek.tunnel",
"processors" : [
{ "remove": { "field": ["host"], "ignore_failure": true } },
{ "set": { "field": "event_type", "value": "zeek.tunnels" } },
{ "pipeline": { "name": "zeek.tunnels" } }
]
}

4
salt/elasticsearch/files/ingest/zeek.tunnels
View File

@@ -12,8 +12,8 @@
{ "rename": { "field": "message2.id.resp_h", "target_field": "destination.ip", "ignore_missing": true } },
{ "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } },
{ "rename": { "field": "message2.tunnel_type", "target_field": "tunnel_type", "ignore_missing": true } },
{ "rename": { "field": "message2.action", "target_field": "action", "ignore_missing": true } },
{ "rename": { "field": "message2.tunnel_type", "target_field": "tunnel.type", "ignore_missing": true } },
{ "rename": { "field": "message2.action", "target_field": "event.action", "ignore_missing": true } },
{ "pipeline": { "name": "zeek.common" } }
]
}

1
salt/elasticsearch/files/ingest/zeek.weird
View File

@@ -2,7 +2,6 @@
"description" : "zeek.weird",
"processors" : [
{ "remove": { "field": ["host"], "ignore_failure": true } },
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "rename": { "field": "message2.name", "target_field": "weird.name", "ignore_missing": true } },
{ "rename": { "field": "message2.addl", "target_field": "weird.additional_info", "ignore_missing": true } },
{ "rename": { "field": "message2.notice", "target_field": "weird.notice", "ignore_missing": true } },