diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index 2d1dde973..f9d6199ba 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -1,17 +1,10 @@ { "description" : "common", "processors" : [ - { - "rename": { - "field": "type", - "target_field": "event_type", - "ignore_missing": true - } - }, { "geoip": { "field": "destination.ip", - "target_field": "destination_geo", + "target_field": "geo", "database_file": "GeoLite2-City.mmdb", "ignore_missing": true, "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] @@ -20,7 +13,7 @@ { "geoip": { "field": "source.ip", - "target_field": "source_geo", + "target_field": "geo", "database_file": "GeoLite2-City.mmdb", "ignore_missing": true, "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] @@ -41,10 +34,12 @@ "ignore_failure": true, "index_name_format": "yyyy.MM.dd" } - }, + }, + { "rename": { "field": "module", "target_field": "event.module", "ignore_missing": true } }, + { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_missing": true } }, { "remove": { - "field": [ "index_name_prefix"], + "field": [ "index_name_prefix", "beat"], "ignore_failure": false } } diff --git a/salt/elasticsearch/files/ingest/ossec.alert b/salt/elasticsearch/files/ingest/ossec.alert new file mode 100644 index 000000000..39362c4ed --- /dev/null +++ b/salt/elasticsearch/files/ingest/ossec.alert @@ -0,0 +1,53 @@ +{ + "description" : "ossec", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.agent", "target_field": "agent", "ignore_missing": true } }, + { "rename": { "field": "message2.data", "target_field": "data", "ignore_missing": true } }, + { "rename": { "field": "message2.decoder", "target_field": "decoder", "ignore_missing": true } }, + { "rename": { "field": "message2.full_log", "target_field": "log.full", "ignore_missing": true } }, + { "rename": { "field": "message2.id", "target_field": "log.id.id", "ignore_missing": true } }, + { "rename": { "field": "message2.location", "target_field": "location", "ignore_missing": true } }, + { "rename": { "field": "message2.manager", "target_field": "manager", "ignore_missing": true } }, + { "rename": { "field": "message2.predecoder", "target_field": "predecoder", "ignore_missing": true } }, + { "rename": { "field": "message2.timestamp", "target_field": "timestamp", "ignore_missing": true } }, + { "rename": { "field": "message2.previous_log", "target_field": "log.previous_log", "ignore_missing": true } }, + { "rename": { "field": "message2.previous_output", "target_field": "log.previous_output", "ignore_missing": true } }, + { "rename": { "field": "message2.rule", "target_field": "rule", "ignore_missing": true } }, + { "rename": { "field": "data.command", "target_field": "command", "ignore_missing": true } }, + { "rename": { "field": "data.dstip", "target_field": "destination.ip", "ignore_missing": true } }, + { "rename": { "field": "data.dstport", "target_field": "destination.port", "ignore_missing": true } }, + { "rename": { "field": "data.dstuser", "target_field": "user.escalated", "ignore_missing": true } }, + { "rename": { "field": "data.srcip", "target_field": "source.ip", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationIp", "target_field": "destination.ip", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationPort", "target_field": "destination.port", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.image", "target_field": "image_path", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.parentImage", "target_field": "parent_image_path", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourceHostname", "target_field": "source.hostname", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourceIp", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourcePort", "target_field": "source.port", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.targetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.user", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "data.win.system.eventID", "target_field": "event.code", "ignore_missing": true } }, + { "rename": { "field": "predecoder.program_name", "target_field": "process.name", "ignore_missing": true } }, + { "set": { "if": "ctx.rule.level == 1", "field": "rule.category", "value": "None" } }, + { "set": { "if": "ctx.rule.level == 2", "field": "rule.category", "value": "System low priority notification" } }, + { "set": { "if": "ctx.rule.level == 3", "field": "rule.category", "value": "Successful/authorized event" } }, + { "set": { "if": "ctx.rule.level == 4", "field": "rule.category", "value": "System low priority error" } }, + { "set": { "if": "ctx.rule.level == 5", "field": "rule.category", "value": "User generated error" } }, + { "set": { "if": "ctx.rule.level == 6", "field": "rule.category", "value": "Low relevance attack" } }, + { "set": { "if": "ctx.rule.level == 7", "field": "rule.category", "value": "\"Bad word\" matching" } }, + { "set": { "if": "ctx.rule.level == 8", "field": "rule.category", "value": "First time seen" } }, + { "set": { "if": "ctx.rule.level == 9", "field": "rule.category", "value": "Error from invalid source" } }, + { "set": { "if": "ctx.rule.level == 10", "field": "rule.category", "value": "Multiple user generated errors" } }, + { "set": { "if": "ctx.rule.level == 11", "field": "rule.category", "value": "Integrity checking warning" } }, + { "set": { "if": "ctx.rule.level == 12", "field": "rule.category", "value": "High importance event" } }, + { "set": { "if": "ctx.rule.level == 13", "field": "rule.category", "value": "Unusal error (high importance)" } }, + { "set": { "if": "ctx.rule.level == 14", "field": "rule.category", "value": "High importance security event" } }, + { "set": { "if": "ctx.rule.level == 15", "field": "rule.category", "value": "Severe attack" } }, + { "append": { "if": "ctx.rule.level != null", "field": "tags", "value": ["alert"] } }, + { "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file new file mode 100644 index 000000000..8652fb912 --- /dev/null +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -0,0 +1,12 @@ +{ + "description" : "strelka", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.file", "target_field": "file", "ignore_missing": true } }, + { "rename": { "field": "message2.scan", "target_field": "scan", "ignore_missing": true } }, + { "rename": { "field": "message2.request", "target_field": "request", "ignore_missing": true } }, + { "rename": { "field": "scan.hash", "target_field": "file.hash", "ignore_missing": true } }, + { "remove": { "field": ["host", "path"], "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert new file mode 100644 index 000000000..d58715f8a --- /dev/null +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -0,0 +1,11 @@ +{ + "description" : "suricata.alert", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename":{ "field": "message2.comunity_id", "target_field": "network.comunity_id", "ignore_failure": true } }, + { "rename":{ "field": "message2.alert", "target_field": "rule", "ignore_failure": true } }, + { "rename":{ "field": "rule.signature", "target_field": "rule.name", "ignore_failure": true } }, + { "rename":{ "field": "rule.ref", "target_field": "rule.version", "ignore_failure": true } }, + { "pipeline": { "name": "suricata.common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common new file mode 100644 index 000000000..79876d366 --- /dev/null +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -0,0 +1,14 @@ +{ + "description" : "suricata.common", + "processors" : [ + { "rename":{ "field": "message2.proto", "target_field": "network.transport", "ignore_failure": true } }, + { "rename":{ "field": "message2.flow_id", "target_field": "event.id", "ignore_failure": true } }, + { "rename":{ "field": "message2.comunity_id", "target_field": "network.comunity_id", "ignore_failure": true } }, + { "rename":{ "field": "message2.src_ip", "target_field": "source.ip", "ignore_failure": true } }, + { "rename":{ "field": "message2.src_port", "target_field": "source.port", "ignore_failure": true } }, + { "rename":{ "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } }, + { "rename":{ "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } }, + { "remove": { "field": ["message2"], "ignore_failure": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index cc854fab0..85902ffa5 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -1,7 +1,7 @@ { "description" : "zeek.common", "processors" : [ - { "rename": { "field": "@timestamp", "target_field": "es.timestamp", "ignore_missing": true } }, + { "rename": { "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, { "rename": { "field": "message2.uid", "target_field": "log.id.uid", "ignore_missing": true } }, { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } }, @@ -15,8 +15,6 @@ { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } }, { "set": { "field": "server.port", "value": "{{destination.port}}" } }, - { "set": { "field": "event.module", "value": "zeek" } }, - { "grok": { "field": "event_type", "patterns": ["zeek.%{WORD:event.dataset}"] } }, { "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } }, { "remove": { "field": ["message2.ts", "path"], "ignore_failure": true } }, { "pipeline": { "name": "common" } } diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index d7878aa0b..3c6da2718 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -1,7 +1,7 @@ { "description" : "zeek.conn", "processors" : [ - { "remove": { "field": ["host"], "ignore_failure": false } }, + { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, { "rename": { "field": "message2.service", "target_field": "network.protocol", "ignore_missing": true } }, @@ -21,20 +21,6 @@ { "rename": { "field": "message2.orig_cc", "target_field": "client.country_code","ignore_missing": true } }, { "rename": { "field": "message2.resp_cc", "target_field": "server.country_code", "ignore_missing": true } }, { "rename": { "field": "message2.sensorname", "target_field": "observer.name", "ignore_missing": true } }, - { "script": { "lang": "painless", "source": "ctx.connection.bytes.total = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } }, - { "set": { "if": "ctx.connection.state == 'S0'", "field": "connection.state_description", "value": "Connection attempt seen, no reply" } }, - { "set": { "if": "ctx.connection.state == 'S1'", "field": "connection.state_description", "value": "Connection established, not terminated" } }, - { "set": { "if": "ctx.connection.state == 'S2'", "field": "connection.state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } }, - { "set": { "if": "ctx.connection.state == 'S3'", "field": "connection.state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } }, - { "set": { "if": "ctx.connection.state == 'SF'", "field": "connection.state_description", "value": "Normal SYN/FIN completion" } }, - { "set": { "if": "ctx.connection.state == 'REJ'", "field": "connection.state_description", "value": "Connection attempt rejected" } }, - { "set": { "if": "ctx.connection.state == 'RSTO'", "field": "connection.state_description", "value": "Connection established, originator aborted (sent a RST)" } }, - { "set": { "if": "ctx.connection.state == 'RSTR'", "field": "connection.state_description", "value": "Established, responder aborted" } }, - { "set": { "if": "ctx.connection.state == 'RSTOS0'","field": "connection.state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } }, - { "set": { "if": "ctx.connection.state == 'RSTRH'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } }, - { "set": { "if": "ctx.connection.state == 'SH'", "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } }, - { "set": { "if": "ctx.connection.state == 'SHR'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } }, - { "set": { "if": "ctx.connection.state == 'OTH'", "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } }, { "pipeline": { "name": "zeek.common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.dpd b/salt/elasticsearch/files/ingest/zeek.dpd index ce46cdc47..59ceed368 100644 --- a/salt/elasticsearch/files/ingest/zeek.dpd +++ b/salt/elasticsearch/files/ingest/zeek.dpd @@ -3,7 +3,6 @@ "processors" : [ { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } }, { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, @@ -13,8 +12,8 @@ { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } }, { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.analyzer", "target_field": "analyzer", "ignore_missing": true } }, - { "rename": { "field": "message2.failure_reason", "target_field": "failure_reason", "ignore_missing": true } }, + { "rename": { "field": "message2.analyzer", "target_field": "observer.analyzer", "ignore_missing": true } }, + { "rename": { "field": "message2.failure_reason", "target_field": "error.reason", "ignore_missing": true } }, { "pipeline": { "name": "zeek.common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.http b/salt/elasticsearch/files/ingest/zeek.http index 1ac3ae42b..a1354044c 100644 --- a/salt/elasticsearch/files/ingest/zeek.http +++ b/salt/elasticsearch/files/ingest/zeek.http @@ -2,7 +2,7 @@ "description" : "zeek.http", "processors" : [ { "remove": { "field": ["host"], "ignore_failure": true } }, - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.trans_depth", "target_field": "http.trans_depth", "ignore_missing": true } }, { "rename": { "field": "message2.method", "target_field": "http.method", "ignore_missing": true } }, { "rename": { "field": "message2.host", "target_field": "http.virtual_host", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tunnel b/salt/elasticsearch/files/ingest/zeek.tunnel index 78eb6eba8..66250091e 100644 --- a/salt/elasticsearch/files/ingest/zeek.tunnel +++ b/salt/elasticsearch/files/ingest/zeek.tunnel @@ -2,7 +2,6 @@ "description" : "zeek.tunnel", "processors" : [ { "remove": { "field": ["host"], "ignore_failure": true } }, - { "set": { "field": "event_type", "value": "zeek.tunnels" } }, { "pipeline": { "name": "zeek.tunnels" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.tunnels b/salt/elasticsearch/files/ingest/zeek.tunnels index 5d8b7f8bf..bcddb61df 100644 --- a/salt/elasticsearch/files/ingest/zeek.tunnels +++ b/salt/elasticsearch/files/ingest/zeek.tunnels @@ -12,8 +12,8 @@ { "rename": { "field": "message2.id.resp_h", "target_field": "destination.ip", "ignore_missing": true } }, { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } }, - { "rename": { "field": "message2.tunnel_type", "target_field": "tunnel_type", "ignore_missing": true } }, - { "rename": { "field": "message2.action", "target_field": "action", "ignore_missing": true } }, + { "rename": { "field": "message2.tunnel_type", "target_field": "tunnel.type", "ignore_missing": true } }, + { "rename": { "field": "message2.action", "target_field": "event.action", "ignore_missing": true } }, { "pipeline": { "name": "zeek.common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.weird b/salt/elasticsearch/files/ingest/zeek.weird index 43d552888..7e2643fe4 100644 --- a/salt/elasticsearch/files/ingest/zeek.weird +++ b/salt/elasticsearch/files/ingest/zeek.weird @@ -2,7 +2,6 @@ "description" : "zeek.weird", "processors" : [ { "remove": { "field": ["host"], "ignore_failure": true } }, - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.name", "target_field": "weird.name", "ignore_missing": true } }, { "rename": { "field": "message2.addl", "target_field": "weird.additional_info", "ignore_missing": true } }, { "rename": { "field": "message2.notice", "target_field": "weird.notice", "ignore_missing": true } },