mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 17:22:49 +01:00
Merge pull request #1498 from Security-Onion-Solutions/feature/filterlog
Feature/filterlog
This commit is contained in:
weslambert
GitHub
@@ -21,44 +21,33 @@
|
||||
"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"split": {
|
||||
"field": "_index",
|
||||
"target_field": "index_name_prefix",
|
||||
"separator": "-"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date_index_name": {
|
||||
"field": "@timestamp",
|
||||
"index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-",
|
||||
"date_rounding": "d",
|
||||
"ignore_failure": true,
|
||||
"index_name_format": "yyyy.MM.dd"
|
||||
}
|
||||
},
|
||||
{ "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } },
|
||||
{ "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } },
|
||||
{ "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } },
|
||||
{ "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } },
|
||||
{ "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "lowercase": { "field": "network.transport", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } },
|
||||
{ "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } },
|
||||
{ "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } },
|
||||
{ "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } },
|
||||
{ "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } },
|
||||
{ "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
|
||||
{
|
||||
"remove": {
|
||||
"field": [ "index_name_prefix", "message2", "type", "fields" ],
|
||||
"ignore_failure": true
|
||||
{ "remove": { "field": [ "message2", "type", "fields" ], "ignore_failure": true } },
|
||||
{
|
||||
"date_index_name": {
|
||||
"field": "@timestamp",
|
||||
"index_name_prefix": "{{ _index }}-",
|
||||
"date_rounding": "d",
|
||||
"ignore_failure": true,
|
||||
"index_name_format": "yyyy.MM.dd"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
59
salt/elasticsearch/files/ingest/filterlog
Normal file
59
salt/elasticsearch/files/ingest/filterlog
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"description" : "filterlog",
|
||||
"processors" : [
|
||||
{
|
||||
"dissect": {
|
||||
"field": "real_message",
|
||||
|
||||
"pattern" : "%{rule.uuid},%{rule.sub_uuid},%{firewall.anchor},%{firewall.tracker_id},%{interface.name},%{rule.reason},%{rule.action},%{network.direction},%{ip.version},%{firewall.sub_message}",
|
||||
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dissect": {
|
||||
"if": "ctx.ip.version == '4'",
|
||||
"field": "firewall.sub_message",
|
||||
"pattern" : "%{ip.tos},%{ip.ecn},%{ip.ttl},%{ip.id},%{ip.offset},%{ip.flags},%{network.transport_id},%{network.transport},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
|
||||
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dissect": {
|
||||
"if": "ctx.ip?.version == '6'",
|
||||
"field": "firewall.sub_message",
|
||||
"pattern" : "%{network.class},%{network.flow_label},%{network.hop_limit},%{network.transport},%{network.transport_id},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
|
||||
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dissect": {
|
||||
"if": "ctx.network?.transport == 'tcp'",
|
||||
"field": "ip_sub_msg",
|
||||
"pattern" : "%{source.port},%{destination.port},%{data.length},%{tcp.flags},",
|
||||
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dissect": {
|
||||
"if": "ctx.protocol == 'udp'",
|
||||
"field": "ip_sub_msg",
|
||||
"pattern" : "%{source.port},%{destination.port},%{data.length}",
|
||||
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"split": {
|
||||
"if": "ctx.ip.version =='6' && ctx.network?.transport == 'Options'",
|
||||
"field": "ip_sub_msg",
|
||||
"target_field": "ip.options",
|
||||
"separator" : ",",
|
||||
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
|
||||
}
|
||||
},
|
||||
{ "set": { "field": "_index", "value": "so-firewall", "override": true } },
|
||||
{ "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } },
|
||||
{ "set": { "field": "module", "value": "pfsense", "override": true } },
|
||||
{ "set": { "field": "dataset", "value": "firewall", "override": true } },
|
||||
{ "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } },
|
||||
]
|
||||
}
|
||||
@@ -12,6 +12,9 @@
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{ "grok": { "field": "message", "patterns": ["<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}"], "ignore_failure": false } },
|
||||
{ "set": { "if": "ctx.source.application == 'filterlog'", "field": "dataset", "value": "firewall" } },
|
||||
{ "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog" } },
|
||||
{ "pipeline": { "name": "common" } }
|
||||
]
|
||||
}
|
||||
|
||||
@@ -257,7 +257,11 @@
|
||||
"type":"object",
|
||||
"dynamic": true
|
||||
},
|
||||
"irc":{
|
||||
"ip":{
|
||||
"type":"object",
|
||||
"dynamic": true
|
||||
},
|
||||
"irc":{
|
||||
"type":"object",
|
||||
"dynamic": true
|
||||
},
|
||||
@@ -273,7 +277,7 @@
|
||||
"type":"object",
|
||||
"dynamic": true
|
||||
},
|
||||
"message":{
|
||||
"message":{
|
||||
"type":"text",
|
||||
"fields":{
|
||||
"keyword":{
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
{%- endif %}
|
||||
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
|
||||
output {
|
||||
if "firewall" in [tags] {
|
||||
if [dataset] =~ "firewall" {
|
||||
elasticsearch {
|
||||
hosts => "{{ ES }}"
|
||||
index => "so-firewall-%{+YYYY.MM.dd}"
|
||||
|
||||
Reference in New Issue
Block a user