diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index 82ab27b2b..39dc84026 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -21,44 +21,33 @@ "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] } }, - { - "split": { - "field": "_index", - "target_field": "index_name_prefix", - "separator": "-" - } - }, - { - "date_index_name": { - "field": "@timestamp", - "index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-", - "date_rounding": "d", - "ignore_failure": true, - "index_name_format": "yyyy.MM.dd" - } - }, - { "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } }, - { "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } }, - { "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } }, - { "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } }, - { "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, - { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, - { "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, - { "lowercase": { "field": "network.transport", "ignore_failure": true, "ignore_missing": true } }, + { "set": { "if": "ctx.event?.severity == 1", "field": "event.severity_label", "value": "low", "override": true } }, + { "set": { "if": "ctx.event?.severity == 2", "field": "event.severity_label", "value": "medium", "override": true } }, + { "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "high", "override": true } }, + { "set": { "if": "ctx.event?.severity == 4", "field": "event.severity_label", "value": "critical", "override": true } }, + { "rename": { "field": "fields.category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, + { "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, - { - "remove": { - "field": [ "index_name_prefix", "message2", "type", "fields" ], - "ignore_failure": true + { "remove": { "field": [ "message2", "type", "fields" ], "ignore_failure": true } }, + { + "date_index_name": { + "field": "@timestamp", + "index_name_prefix": "{{ _index }}-", + "date_rounding": "d", + "ignore_failure": true, + "index_name_format": "yyyy.MM.dd" } } ] +} + ] } diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog new file mode 100644 index 000000000..206e487da --- /dev/null +++ b/salt/elasticsearch/files/ingest/filterlog @@ -0,0 +1,59 @@ +{ + "description" : "filterlog", + "processors" : [ + { + "dissect": { + "field": "real_message", + + "pattern" : "%{rule.uuid},%{rule.sub_uuid},%{firewall.anchor},%{firewall.tracker_id},%{interface.name},%{rule.reason},%{rule.action},%{network.direction},%{ip.version},%{firewall.sub_message}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.ip.version == '4'", + "field": "firewall.sub_message", + "pattern" : "%{ip.tos},%{ip.ecn},%{ip.ttl},%{ip.id},%{ip.offset},%{ip.flags},%{network.transport_id},%{network.transport},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.ip?.version == '6'", + "field": "firewall.sub_message", + "pattern" : "%{network.class},%{network.flow_label},%{network.hop_limit},%{network.transport},%{network.transport_id},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.network?.transport == 'tcp'", + "field": "ip_sub_msg", + "pattern" : "%{source.port},%{destination.port},%{data.length},%{tcp.flags},", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "dissect": { + "if": "ctx.protocol == 'udp'", + "field": "ip_sub_msg", + "pattern" : "%{source.port},%{destination.port},%{data.length}", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "split": { + "if": "ctx.ip.version =='6' && ctx.network?.transport == 'Options'", + "field": "ip_sub_msg", + "target_field": "ip.options", + "separator" : ",", + "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] + } + }, + { "set": { "field": "_index", "value": "so-firewall", "override": true } }, + { "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } }, + { "set": { "field": "module", "value": "pfsense", "override": true } }, + { "set": { "field": "dataset", "value": "firewall", "override": true } }, + { "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } }, + ] +} diff --git a/salt/elasticsearch/files/ingest/syslog b/salt/elasticsearch/files/ingest/syslog index 1af0bc1c8..b4e09e9df 100644 --- a/salt/elasticsearch/files/ingest/syslog +++ b/salt/elasticsearch/files/ingest/syslog @@ -12,6 +12,9 @@ "ignore_failure": true } }, + { "grok": { "field": "message", "patterns": ["<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}"], "ignore_failure": false } }, + { "set": { "if": "ctx.source.application == 'filterlog'", "field": "dataset", "value": "firewall" } }, + { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog" } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 43a4c7378..cc4c4595b 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -257,7 +257,11 @@ "type":"object", "dynamic": true }, - "irc":{ + "ip":{ + "type":"object", + "dynamic": true + }, + "irc":{ "type":"object", "dynamic": true }, @@ -273,7 +277,7 @@ "type":"object", "dynamic": true }, - "message":{ + "message":{ "type":"text", "fields":{ "keyword":{ diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index f8aa07b1b..3ad4a5722 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -5,7 +5,7 @@ {%- endif %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { - if "firewall" in [tags] { + if [dataset] =~ "firewall" { elasticsearch { hosts => "{{ ES }}" index => "so-firewall-%{+YYYY.MM.dd}"