From b8137214e4b4b7f690f8c2a9e8b3fcd05d416e6a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 26 Feb 2021 08:08:09 -0500 Subject: [PATCH 1/2] Initial Support - Live Query to Hunt --- .../config/so/0008_input_redis.conf.jinja | 17 ++++++++ .../config/so/9001_output_osq.conf.jinja | 41 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja create mode 100644 salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja diff --git a/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja new file mode 100644 index 000000000..694b997bb --- /dev/null +++ b/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja @@ -0,0 +1,17 @@ +{%- set MANAGER = salt['grains.get']('master') %} +{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} +{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} + +## TO DO - Add this to Logstash Pipeline Pillar - Manager + +input { + redis { + host => '{{ MANAGER }}' + port => 6379 + data_type => 'pattern_channel' + key => 'results_*' + type => 'osq' + threads => {{ THREADS }} + batch_count => {{ BATCH }} + } +} diff --git a/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja b/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja new file mode 100644 index 000000000..6a4c564f0 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja @@ -0,0 +1,41 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} + +## TO DO - Add this to Logstash Pipeline Pillar - Search + +filter { + if [type] =~ "osq" { + + split { + field => "rows" + } + mutate { + rename => { + "[rows][cmdline]" => "[process][commandline]" + "[rows][name]" => "[process][name]" + } + } + } +} + + +output { + if [type] =~ "osq" { + elasticsearch { + pipeline => "common" + hosts => "{{ ES }}" + index => "so-osquery" + template_name => "so-osquery" + template => "/templates/so-osquery-template.json" + template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} + } + } +} From 548f67ca6f8a29c94240a82cf99464dc28e2b6d6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 4 Mar 2021 18:21:13 -0500 Subject: [PATCH 2/2] Initial support for Live Queries in Hunt --- pillar/logstash/manager.sls | 1 + pillar/logstash/search.sls | 1 + .../files/ingest/osquery.live_query | 16 +++++++++++++ .../files/ingest/osquery.normalize | 14 +++++++++++ .../files/ingest/osquery.query_result | 17 +++++-------- .../templates/so/so-common-template.json | 4 ++++ ... => 0008_input_fleet_livequery.conf.jinja} | 8 ++++--- .../config/so/9100_output_osquery.conf.jinja | 2 +- ... 9101_output_osquery_livequery.conf.jinja} | 24 ++++++++++--------- salt/soc/files/soc/hunt.queries.json | 1 + 10 files changed, 62 insertions(+), 26 deletions(-) create mode 100644 salt/elasticsearch/files/ingest/osquery.live_query create mode 100644 salt/elasticsearch/files/ingest/osquery.normalize rename salt/logstash/pipelines/config/so/{0008_input_redis.conf.jinja => 0008_input_fleet_livequery.conf.jinja} (74%) rename salt/logstash/pipelines/config/so/{9001_output_osq.conf.jinja => 9101_output_osquery_livequery.conf.jinja} (72%) diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index 6f3ba495b..1aa445c97 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -3,6 +3,7 @@ logstash: pipelines: manager: config: + - so/0008_input_fleet_livequery.conf.jinja - so/0009_input_beats.conf - so/0010_input_hhbeats.conf - so/9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 7a5aeec39..8ae84fe1f 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -8,6 +8,7 @@ logstash: - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja - so/9100_output_osquery.conf.jinja + - so/9101_output_osquery_livequery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja diff --git a/salt/elasticsearch/files/ingest/osquery.live_query b/salt/elasticsearch/files/ingest/osquery.live_query new file mode 100644 index 000000000..92f325e1d --- /dev/null +++ b/salt/elasticsearch/files/ingest/osquery.live_query @@ -0,0 +1,16 @@ +{ + "description" : "osquery live query", + "processors" : [ + { + "script": { + "lang": "painless", + "source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; " + } + }, + { "remove": { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } }, + { "rename": { "field": "distributed_query_execution_id", "target_field": "result.query_id", "ignore_missing": true } }, + { "rename": { "field": "computer_name", "target_field": "host.hostname", "ignore_missing": true } }, + { "pipeline": { "name": "osquery.normalize" } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/osquery.normalize b/salt/elasticsearch/files/ingest/osquery.normalize new file mode 100644 index 000000000..ce0a6ca92 --- /dev/null +++ b/salt/elasticsearch/files/ingest/osquery.normalize @@ -0,0 +1,14 @@ +{ + "description" : "osquery normalize", + "processors" : [ + { "rename": { "field": "result.columns.cmdline", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "result.columns.cwd", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "result.columns.name", "target_field": "process.name", "ignore_missing": true } }, + { "rename": { "field": "result.columns.path", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "result.columns.pid", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "result.columns.parent", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "result.columns.uid", "target_field": "user.id", "ignore_missing": true } }, + { "rename": { "field": "result.columns.username", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "result.columns.gid", "target_field": "group.id", "ignore_missing": true } } + ] +} diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result index b6b4f22ef..9bb381946 100644 --- a/salt/elasticsearch/files/ingest/osquery.query_result +++ b/salt/elasticsearch/files/ingest/osquery.query_result @@ -1,24 +1,19 @@ { "description" : "osquery", "processors" : [ - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, - { "gsub": { "field": "message2.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } }, - { "rename": { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns", "target_field": "winlog", "ignore_missing": true } }, + { "json": { "field": "message", "target_field": "result", "ignore_failure": true } }, + { "gsub": { "field": "result.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } }, + { "rename": { "if": "ctx.result.columns?.eventid != null", "field": "result.columns", "target_field": "winlog", "ignore_missing": true } }, { "json": { "field": "winlog.data", "target_field": "unparsed", "ignore_failure": true} }, { "set": { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true } }, { "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } }, { "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } }, { "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, - { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { - "script": { - "lang": "painless", - "source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; " - } - }, + { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } }, { "set": { "field": "event.module", "value": "osquery", "override": false } }, - { "set": { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false} }, + { "set": { "field": "event.dataset", "value": "{{result.name}}", "override": false} }, + { "pipeline": { "if": "!(ctx.containsKey('winlog'))", "name": "osquery.normalize" } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 062838670..012c590d0 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -365,6 +365,10 @@ "request":{ "type":"object", "dynamic": true + }, + "result":{ + "type":"object", + "dynamic": true }, "rfb":{ "type":"object", diff --git a/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja similarity index 74% rename from salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja rename to salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja index 694b997bb..83aa0c02d 100644 --- a/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja @@ -2,15 +2,17 @@ {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} -## TO DO - Add this to Logstash Pipeline Pillar - Manager - input { redis { host => '{{ MANAGER }}' port => 6379 data_type => 'pattern_channel' key => 'results_*' - type => 'osq' + type => 'live_query' + add_field => { + "module" => "osquery" + "dataset" => "live_query" + } threads => {{ THREADS }} batch_count => {{ BATCH }} } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 2a71e3fab..9e660f8a8 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -5,7 +5,7 @@ {%- endif %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { - if [module] =~ "osquery" { + if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" diff --git a/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja similarity index 72% rename from salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja rename to salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja index 6a4c564f0..51e691176 100644 --- a/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -5,28 +5,30 @@ {%- endif %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} -## TO DO - Add this to Logstash Pipeline Pillar - Search - filter { - if [type] =~ "osq" { + if [type] =~ "live_query" { + mutate { + rename => { + "[host][hostname]" => "computer_name" + } + } + + prune { + blacklist_names => ["host"] + } + split { field => "rows" } - mutate { - rename => { - "[rows][cmdline]" => "[process][commandline]" - "[rows][name]" => "[process][name]" - } - } } } output { - if [type] =~ "osq" { + if [type] =~ "live_query" { elasticsearch { - pipeline => "common" + pipeline => "osquery.live_query" hosts => "{{ ES }}" index => "so-osquery" template_name => "so-osquery" diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index b8dc5eb21..5f3a359b5 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -42,6 +42,7 @@ { "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.dataset:mysql | groupby mysql.command"}, { "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.dataset:notice | groupby notice.note notice.message"}, { "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.dataset:ntlm | groupby ntlm.server.dns.name"}, + { "name": "Osquery Live Queries", "description": "Osquery Live Query results grouped by computer name", "query": "event.dataset:live_query | groupby host.hostname"}, { "name": "PE", "description": "PE files list", "query": "event.dataset:pe | groupby file.machine file.os file.subsystem"}, { "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.dataset:radius | groupby user.name.keyword"}, { "name": "RDP", "description": "RDP grouped by client name", "query": "event.dataset:rdp | groupby client.name"},