diff --git a/salt/elasticsearch/files/ingest/syslog b/salt/elasticsearch/files/ingest/syslog index bf40efec5..8919c3e1f 100644 --- a/salt/elasticsearch/files/ingest/syslog +++ b/salt/elasticsearch/files/ingest/syslog @@ -1,36 +1,157 @@ { - "description" : "syslog", + "description" : "syslog pipeline", "processors" : [ { - "dissect": { - "field": "message", - "pattern" : "%{message}", - "on_failure": [ { "drop" : { } } ] - }, - "remove": { - "field": [ "type", "agent" ], - "ignore_failure": true - } + "dissect": { + "field": "message", + "pattern" : "%{message}", + "on_failure": [ { "drop" : { } } ] + }, + "remove": { + "field": [ "type", "agent" ], + "ignore_failure": true + } + }, { + "grok": { + "field": "message", + "patterns": [ + "^<%{INT:syslog.priority:int}>%{TIMESTAMP_ISO8601:syslog.timestamp} +%{IPORHOST:syslog.host} +%{PROG:syslog.program}(?:\\[%{POSINT:syslog.pid:int}\\])?: %{GREEDYDATA:real_message}$", + + "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$", + + "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$" + ], + "ignore_failure": true + } }, { - "grok": - { - "field": "message", - "patterns": [ - "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$", - "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$" - ], - "ignore_failure": true - } + "convert" : { + "if": "ctx?.syslog?.priority != null", + "field" : "syslog.priority", + "type": "integer" + } }, - { "set": { "if": "ctx.source?.application == 'filterlog'", "field": "dataset", "value": "firewall", "ignore_failure": true } }, - { "set": { "if": "ctx.vendor != null", "field": "module", "value": "{{ vendor }}", "ignore_failure": true } }, - { "set": { "if": "ctx.product != null", "field": "dataset", "value": "{{ product }}", "ignore_failure": true } }, - { "set": { "field": "event.ingested", "value": "{{ @timestamp }}" } }, - { "date": { "if": "ctx.syslog?.timestamp != null", "field": "syslog.timestamp", "target_field": "@timestamp", "formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], "ignore_failure": true } }, - { "remove": { "field": ["pid", "program"], "ignore_missing": true, "ignore_failure": true } }, - { "pipeline": { "if": "ctx.vendor != null && ctx.product != null", "name": "{{ vendor }}.{{ product }}", "ignore_failure": true } }, - { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog", "ignore_failure": true } }, - { "pipeline": { "name": "common" } } + { + "script": { + "description": "Map syslog priority into facility and level", + "lang": "painless", + "params" : { + "level": [ + "emerg", + "alert", + "crit", + "err", + "warn", + "notice", + "info", + "debug" + ], + "facility" : [ + "kern", + "user", + "mail", + "daemon", + "auth", + "syslog", + "lpr", + "news", + "uucp", + "cron", + "authpriv", + "ftp", + "ntp", + "security", + "console", + "solaris-cron", + "local0", + "local1", + "local2", + "local3", + "local4", + "local5", + "local6", + "local7" + ] + }, + "source": "if (ctx['syslog'] != null && ctx['syslog']['priority'] != null) { int p = ctx['syslog']['priority']; int f = p / 8; int l = p - (f * 8); ctx['syslog']['facility_label'] = [ : ]; ctx['syslog']['severity_label'] = [ : ]; ctx['syslog'].put('severity', l); ctx['syslog'].put('severity_label', params.level[l].toUpperCase()); ctx['syslog'].put('facility', f); ctx['syslog'].put('facility_label', params.facility[f].toUpperCase()); }" + + } + }, + { + "set": { + "if": "ctx.syslog?.host != null", + "field": "host.name", + "value": "{{ syslog.host }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.syslog?.program != null", + "field": "process.name", + "value": "{{ syslog.program }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.syslog?.pid != null", + "field": "process.id", + "value": "{{ syslog.pid }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.source?.application == 'filterlog'", + "field": "dataset", + "value": "firewall", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.vendor != null", + "field": "module", + "value": "{{ vendor }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.product != null", + "field": "dataset", + "value": "{{ product }}", + "ignore_failure": true + } + }, { + "set": { + "field": "ingest.timestamp", + "value": "{{ @timestamp }}" + } + }, { + "date": { + "if": "ctx.syslog?.timestamp != null", + "field": "syslog.timestamp", + "target_field": "@timestamp", + "formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], + "ignore_failure": true + } + }, { + "remove": { + "field": ["pid", "program"], + "ignore_missing": true, + "ignore_failure": true + } + }, { + "pipeline": { + "if": "ctx.vendor != null && ctx.product != null", + "name": "{{ vendor }}.{{ product }}", + "ignore_failure": true + } + }, { + "pipeline": { + "if": "ctx.dataset == 'firewall'", + "name": "filterlog", + "ignore_failure": true + } + }, { + "pipeline": { "name": "common" } + } ] }