From f21e52b431efc720ae05b717ef60e048345daa27 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Jan 2019 11:11:29 -0500 Subject: [PATCH 1/5] Add missing character --- salt/logstash/files/dynamic/0006_input_beats.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/logstash/files/dynamic/0006_input_beats.conf b/salt/logstash/files/dynamic/0006_input_beats.conf index c7cab30b7..b9f03c979 100644 --- a/salt/logstash/files/dynamic/0006_input_beats.conf +++ b/salt/logstash/files/dynamic/0006_input_beats.conf @@ -25,7 +25,7 @@ filter { add_field => { "syslog-host_from" => "%{[beat][name]}" } remove_field => [ "beat", "prospector", "input", "offset" ] } - + } if [type] == "osquery" { mutate { remove_tag => ["beat"] @@ -35,6 +35,5 @@ filter { source => "message" target => "osquery" } - } } From 15bfce07e8e3447b9b27f837017737d63f94cf85 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Jan 2019 11:13:05 -0500 Subject: [PATCH 2/5] Move osquery parsers from custom to dynamic --- .../files/dynamic/7100_osquery_wel.conf | 23 +++++++++++++++++++ .../files/dynamic/9100_output_osquery.conf | 19 +++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 salt/logstash/files/dynamic/7100_osquery_wel.conf create mode 100644 salt/logstash/files/dynamic/9100_output_osquery.conf diff --git a/salt/logstash/files/dynamic/7100_osquery_wel.conf b/salt/logstash/files/dynamic/7100_osquery_wel.conf new file mode 100644 index 000000000..b4d77d83f --- /dev/null +++ b/salt/logstash/files/dynamic/7100_osquery_wel.conf @@ -0,0 +1,23 @@ +# Author: Josh Brower +# Last Update: 12/28/2018 +# If log is tagged osquery and there is an eventid column - then cleanup and parse out the EventData column + +filter { + if "osquery" in [tags] and [osquery][columns][eventid] { + + mutate { + gsub => ["[osquery][columns][data]", "\\x0A", ""] + } + + json { + source => "[osquery][columns][data]" + target => "[osquery][columns][data]" + } + + mutate { + merge => { "[osquery][columns]" => "[osquery][columns][data]" } + remove_field => ["[osquery][columns][data]"] + } + + } +} \ No newline at end of file diff --git a/salt/logstash/files/dynamic/9100_output_osquery.conf b/salt/logstash/files/dynamic/9100_output_osquery.conf new file mode 100644 index 000000000..e95119562 --- /dev/null +++ b/salt/logstash/files/dynamic/9100_output_osquery.conf @@ -0,0 +1,19 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} +# Author: Josh Brower +# Last Update: 12/29/2018 +# Output to ES for osquery tagged logs + + +output { + if "osquery" in [tags] { + elasticsearch { + hosts => "{{ ES }}" + index => "logstash-osquery-%{+YYYY.MM.dd}" + template => "/logstash-template.json" + } + } +} \ No newline at end of file From bc7bf5774a12491d85c20ae4db54a3234b3781c8 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Jan 2019 11:14:38 -0500 Subject: [PATCH 3/5] Enable osquery parsers for EVAL role --- salt/logstash/conf/conf.enabled.txt.so-eval | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index 71e50525f..5c85ab8ac 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -107,3 +107,5 @@ /usr/share/logstash/pipeline.dynamic/9500_output_beats.conf /usr/share/logstash/pipeline.dynamic/9600_output_ossec.conf /usr/share/logstash/pipeline.dynamic/9998_output_test_data.conf +/usr/share/logstash/pipeline.dynamic/7100_osquery_wel.conf +/usr/share/logstash/pipeline.dynamic/9100_output_osquery.conf From b9f62699257154aaacec758fbb8f951ae0b21b92 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Jan 2019 11:20:01 -0500 Subject: [PATCH 4/5] Moved to dynamic --- .../custom/parsers/7100_osquery_wel.conf | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 salt/logstash/files/custom/parsers/7100_osquery_wel.conf diff --git a/salt/logstash/files/custom/parsers/7100_osquery_wel.conf b/salt/logstash/files/custom/parsers/7100_osquery_wel.conf deleted file mode 100644 index 3dea60269..000000000 --- a/salt/logstash/files/custom/parsers/7100_osquery_wel.conf +++ /dev/null @@ -1,23 +0,0 @@ -# Author: Josh Brower -# Last Update: 12/28/2018 -# If log is tagged osquery and there is an eventid column, then cleanup and parse out the EventData column - -filter { - if "osquery" in [tags] and [osquery][columns][eventid] { - - mutate { - gsub => ["[osquery][columns][data]", "\\x0A", ""] - } - - json { - source => "[osquery][columns][data]" - target => "[osquery][columns][data]" - } - - mutate { - merge => { "[osquery][columns]" => "[osquery][columns][data]" } - remove_field => ["[osquery][columns][data]"] - } - - } -} From ce43fd7cd436344d563dbfdc7152d64a7c4071a7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Jan 2019 11:20:09 -0500 Subject: [PATCH 5/5] Moved to dynamic --- .../custom/parsers/9100_output_osquery.conf | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 salt/logstash/files/custom/parsers/9100_output_osquery.conf diff --git a/salt/logstash/files/custom/parsers/9100_output_osquery.conf b/salt/logstash/files/custom/parsers/9100_output_osquery.conf deleted file mode 100644 index 59c16347b..000000000 --- a/salt/logstash/files/custom/parsers/9100_output_osquery.conf +++ /dev/null @@ -1,19 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Josh Brower -# Last Update: 12/28/2018 -# Output to ES for osquery tagged logs - - -output { - if "osquery" in [tags] { - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-osquery-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -}