CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1498 from Security-Onion-Solutions/feature/filterlog

Browse Source 
Feature/filterlog
This commit is contained in:
weslambert
2020-10-09 20:05:21 -04:00
committed by GitHub
parent 930ec33cb7 28a1f7f88a
commit 12c3c351d8
5 changed files with 90 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
salt/elasticsearch/files/ingest/common
View File

@@ -21,44 +21,33 @@
"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
}
},
{
"split": {
"field": "_index",
"target_field": "index_name_prefix",
"separator": "-"
}
},
{
"date_index_name": {
"field": "@timestamp",
"index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-",
"date_rounding": "d",
"ignore_failure": true,
"index_name_format": "yyyy.MM.dd"
}
},
{ "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } },
{ "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } },
{ "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } },
{ "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } },
{ "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } },
{ "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
{ "lowercase": { "field": "network.transport", "ignore_failure": true, "ignore_missing": true } },
{ "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } },
{ "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } },
{ "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } },
{ "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } },
{ "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } },
{ "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
{
"remove": {
"field": [ "index_name_prefix", "message2", "type", "fields" ],
"ignore_failure": true
{ "remove": { "field": [ "message2", "type", "fields" ], "ignore_failure": true } },
{
"date_index_name": {
"field": "@timestamp",
"index_name_prefix": "{{ _index }}-",
"date_rounding": "d",
"ignore_failure": true,
"index_name_format": "yyyy.MM.dd"
}
}
]
}
]
}

59
salt/elasticsearch/files/ingest/filterlog Normal file
View File

@@ -0,0 +1,59 @@
{
"description" : "filterlog",
"processors" : [
{
"dissect": {
"field": "real_message",
"pattern" : "%{rule.uuid},%{rule.sub_uuid},%{firewall.anchor},%{firewall.tracker_id},%{interface.name},%{rule.reason},%{rule.action},%{network.direction},%{ip.version},%{firewall.sub_message}",
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
}
},
{
"dissect": {
"if": "ctx.ip.version == '4'",
"field": "firewall.sub_message",
"pattern" : "%{ip.tos},%{ip.ecn},%{ip.ttl},%{ip.id},%{ip.offset},%{ip.flags},%{network.transport_id},%{network.transport},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
}
},
{
"dissect": {
"if": "ctx.ip?.version == '6'",
"field": "firewall.sub_message",
"pattern" : "%{network.class},%{network.flow_label},%{network.hop_limit},%{network.transport},%{network.transport_id},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
}
},
{
"dissect": {
"if": "ctx.network?.transport == 'tcp'",
"field": "ip_sub_msg",
"pattern" : "%{source.port},%{destination.port},%{data.length},%{tcp.flags},",
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
}
},
{
"dissect": {
"if": "ctx.protocol == 'udp'",
"field": "ip_sub_msg",
"pattern" : "%{source.port},%{destination.port},%{data.length}",
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
}
},
{
"split": {
"if": "ctx.ip.version =='6' && ctx.network?.transport == 'Options'",
"field": "ip_sub_msg",
"target_field": "ip.options",
"separator" : ",",
"on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
}
},
{ "set": { "field": "_index", "value": "so-firewall", "override": true } },
{ "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } },
{ "set": { "field": "module", "value": "pfsense", "override": true } },
{ "set": { "field": "dataset", "value": "firewall", "override": true } },
{ "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } },
]
}

3
salt/elasticsearch/files/ingest/syslog
View File

@@ -12,6 +12,9 @@
"ignore_failure": true
}
},
{ "grok": { "field": "message", "patterns": ["<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}"], "ignore_failure": false } },
{ "set": { "if": "ctx.source.application == 'filterlog'", "field": "dataset", "value": "firewall" } },
{ "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog" } },
{ "pipeline": { "name": "common" } }
]
}

8
salt/elasticsearch/templates/so/so-common-template.json
View File

@@ -257,7 +257,11 @@
"type":"object",
"dynamic": true
},
"irc":{
"ip":{
"type":"object",
"dynamic": true
},
"irc":{
"type":"object",
"dynamic": true
},
@@ -273,7 +277,7 @@
"type":"object",
"dynamic": true
},
"message":{
"message":{
"type":"text",
"fields":{
"keyword":{

2
salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
View File

@@ -5,7 +5,7 @@
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if "firewall" in [tags] {
if [dataset] =~ "firewall" {
elasticsearch {
hosts => "{{ ES }}"
index => "so-firewall-%{+YYYY.MM.dd}"