From 69a04dedd33c6d80ff774403e0f29c49d46b90de Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 9 Oct 2020 23:56:52 +0000 Subject: [PATCH 1/3] Filterlog config changes --- salt/elasticsearch/files/ingest/common | 53 +++++++--------- salt/elasticsearch/files/ingest/filterlog | 60 +++++++++++++++++++ salt/elasticsearch/files/ingest/syslog | 3 + .../config/so/9200_output_firewall.conf.jinja | 2 +- 4 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 salt/elasticsearch/files/ingest/filterlog diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index 82ab27b2b..39dc84026 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -21,44 +21,33 @@ "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] } }, - { - "split": { - "field": "_index", - "target_field": "index_name_prefix", - "separator": "-" - } - }, - { - "date_index_name": { - "field": "@timestamp", - "index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-", - "date_rounding": "d", - "ignore_failure": true, - "index_name_format": "yyyy.MM.dd" - } - }, - { "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } }, - { "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } }, - { "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } }, - { "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } }, - { "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, - { "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, - { "lowercase": { "field": "network.transport", "ignore_failure": true, "ignore_missing": true } }, + { "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } }, + { "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } }, + { "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } }, + { "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } }, + { "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, + { "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, - { - "remove": { - "field": [ "index_name_prefix", "message2", "type", "fields" ], - "ignore_failure": true + { "remove": { "field": [ "message2", "type", "fields" ], "ignore_failure": true } }, + { + "date_index_name": { + "field": "@timestamp", + "index_name_prefix": "{{ _index }}-", + "date_rounding": "d", + "ignore_failure": true, + "index_name_format": "yyyy.MM.dd" } } ] +} + ] } diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog new file mode 100644 index 000000000..d40874205 --- /dev/null +++ b/salt/elasticsearch/files/ingest/filterlog @@ -0,0 +1,60 @@ +{ + "description" : "filterlog", + "processors" : [ + { + "dissect": { + "field": "real_message", + + "pattern" : "%{rule.uuid},%{rule.sub_uuid},%{firewall.anchor},%{firewall.tracker_id},%{interface.name},%{rule.reason},%{rule.action},%{network.direction},%{ip.version},%{firewall.sub_message}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.ip.version == '4'", + "field": "firewall.sub_message", + "pattern" : "%{ip.tos},%{ip.ecn},%{ip.ttl},%{ip.id},%{ip.offset},%{ip.flags},%{network.transport_id},%{network.transport},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.ip?.version == '6'", + "field": "firewall.sub_message", + "pattern" : "%{network.class},%{network.flow_label},%{network.hop_limit},%{network.transport},%{network.transport_id},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.network?.transport == 'tcp'", + "field": "ip_sub_msg", + "pattern" : "%{source.port},%{destination.port},%{data.length},%{tcp.flags},", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.protocol == 'udp'", + "field": "ip_sub_msg", + "pattern" : "%{source.port},%{destination.port},%{data.length}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "split": { + "if": "ctx.ip.version =='6' && ctx.network?.transport == 'Options'", + "field": "ip_sub_msg", + "target_field": "ip.options", + "separator" : ",", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { "set": { "field": "_index", "value": "so-firewall", "override": true } }, + { "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } }, + { "set": { "field": "event.module", "value": "pfsense", "override": true } }, + { "set": { "field": "event.dataset", "value": "firewall", "override": true } }, + { "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } }, + { "append": { "field": "tags", "value": ["pfsense"] } } + ] +} diff --git a/salt/elasticsearch/files/ingest/syslog b/salt/elasticsearch/files/ingest/syslog index 1af0bc1c8..b4e09e9df 100644 --- a/salt/elasticsearch/files/ingest/syslog +++ b/salt/elasticsearch/files/ingest/syslog @@ -12,6 +12,9 @@ "ignore_failure": true } }, + { "grok": { "field": "message", "patterns": ["<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}"], "ignore_failure": false } }, + { "set": { "if": "ctx.source.application == 'filterlog'", "field": "dataset", "value": "firewall" } }, + { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog" } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index f8aa07b1b..3ad4a5722 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -5,7 +5,7 @@ {%- endif %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { - if "firewall" in [tags] { + if [dataset] =~ "firewall" { elasticsearch { hosts => "{{ ES }}" index => "so-firewall-%{+YYYY.MM.dd}" From b55ffa44f8535736b7321c153dbc2b8fb1f63a97 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Sat, 10 Oct 2020 00:01:37 +0000 Subject: [PATCH 2/3] Fix module,dataset rename --- salt/elasticsearch/files/ingest/filterlog | 4 ++-- salt/elasticsearch/templates/so/so-common-template.json | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog index d40874205..d7bda9539 100644 --- a/salt/elasticsearch/files/ingest/filterlog +++ b/salt/elasticsearch/files/ingest/filterlog @@ -52,8 +52,8 @@ }, { "set": { "field": "_index", "value": "so-firewall", "override": true } }, { "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } }, - { "set": { "field": "event.module", "value": "pfsense", "override": true } }, - { "set": { "field": "event.dataset", "value": "firewall", "override": true } }, + { "set": { "field": "module", "value": "pfsense", "override": true } }, + { "set": { "field": "dataset", "value": "firewall", "override": true } }, { "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } }, { "append": { "field": "tags", "value": ["pfsense"] } } ] diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 43a4c7378..cc4c4595b 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -257,7 +257,11 @@ "type":"object", "dynamic": true }, - "irc":{ + "ip":{ + "type":"object", + "dynamic": true + }, + "irc":{ "type":"object", "dynamic": true }, @@ -273,7 +277,7 @@ "type":"object", "dynamic": true }, - "message":{ + "message":{ "type":"text", "fields":{ "keyword":{ From 28a1f7f88abd3e03edc67d081a3786c2e3a968c7 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Sat, 10 Oct 2020 00:03:51 +0000 Subject: [PATCH 3/3] Remove pfsense tag --- salt/elasticsearch/files/ingest/filterlog | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog index d7bda9539..206e487da 100644 --- a/salt/elasticsearch/files/ingest/filterlog +++ b/salt/elasticsearch/files/ingest/filterlog @@ -55,6 +55,5 @@ { "set": { "field": "module", "value": "pfsense", "override": true } }, { "set": { "field": "dataset", "value": "firewall", "override": true } }, { "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } }, - { "append": { "field": "tags", "value": ["pfsense"] } } ] }