diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index 6f3ba495b..1aa445c97 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -3,6 +3,7 @@ logstash: pipelines: manager: config: + - so/0008_input_fleet_livequery.conf.jinja - so/0009_input_beats.conf - so/0010_input_hhbeats.conf - so/9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 7a5aeec39..8ae84fe1f 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -8,6 +8,7 @@ logstash: - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja - so/9100_output_osquery.conf.jinja + - so/9101_output_osquery_livequery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja diff --git a/salt/elasticsearch/files/ingest/osquery.live_query b/salt/elasticsearch/files/ingest/osquery.live_query new file mode 100644 index 000000000..92f325e1d --- /dev/null +++ b/salt/elasticsearch/files/ingest/osquery.live_query @@ -0,0 +1,16 @@ +{ + "description" : "osquery live query", + "processors" : [ + { + "script": { + "lang": "painless", + "source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; " + } + }, + { "remove": { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } }, + { "rename": { "field": "distributed_query_execution_id", "target_field": "result.query_id", "ignore_missing": true } }, + { "rename": { "field": "computer_name", "target_field": "host.hostname", "ignore_missing": true } }, + { "pipeline": { "name": "osquery.normalize" } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/osquery.normalize b/salt/elasticsearch/files/ingest/osquery.normalize new file mode 100644 index 000000000..ce0a6ca92 --- /dev/null +++ b/salt/elasticsearch/files/ingest/osquery.normalize @@ -0,0 +1,14 @@ +{ + "description" : "osquery normalize", + "processors" : [ + { "rename": { "field": "result.columns.cmdline", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "result.columns.cwd", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "result.columns.name", "target_field": "process.name", "ignore_missing": true } }, + { "rename": { "field": "result.columns.path", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "result.columns.pid", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "result.columns.parent", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "result.columns.uid", "target_field": "user.id", "ignore_missing": true } }, + { "rename": { "field": "result.columns.username", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "result.columns.gid", "target_field": "group.id", "ignore_missing": true } } + ] +} diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result index b6b4f22ef..9bb381946 100644 --- a/salt/elasticsearch/files/ingest/osquery.query_result +++ b/salt/elasticsearch/files/ingest/osquery.query_result @@ -1,24 +1,19 @@ { "description" : "osquery", "processors" : [ - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, - { "gsub": { "field": "message2.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } }, - { "rename": { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns", "target_field": "winlog", "ignore_missing": true } }, + { "json": { "field": "message", "target_field": "result", "ignore_failure": true } }, + { "gsub": { "field": "result.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } }, + { "rename": { "if": "ctx.result.columns?.eventid != null", "field": "result.columns", "target_field": "winlog", "ignore_missing": true } }, { "json": { "field": "winlog.data", "target_field": "unparsed", "ignore_failure": true} }, { "set": { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true } }, { "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } }, { "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } }, { "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, - { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { - "script": { - "lang": "painless", - "source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; " - } - }, + { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } }, { "set": { "field": "event.module", "value": "osquery", "override": false } }, - { "set": { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false} }, + { "set": { "field": "event.dataset", "value": "{{result.name}}", "override": false} }, + { "pipeline": { "if": "!(ctx.containsKey('winlog'))", "name": "osquery.normalize" } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 062838670..012c590d0 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -365,6 +365,10 @@ "request":{ "type":"object", "dynamic": true + }, + "result":{ + "type":"object", + "dynamic": true }, "rfb":{ "type":"object", diff --git a/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja new file mode 100644 index 000000000..83aa0c02d --- /dev/null +++ b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja @@ -0,0 +1,19 @@ +{%- set MANAGER = salt['grains.get']('master') %} +{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} +{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} + +input { + redis { + host => '{{ MANAGER }}' + port => 6379 + data_type => 'pattern_channel' + key => 'results_*' + type => 'live_query' + add_field => { + "module" => "osquery" + "dataset" => "live_query" + } + threads => {{ THREADS }} + batch_count => {{ BATCH }} + } +} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 2a71e3fab..9e660f8a8 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -5,7 +5,7 @@ {%- endif %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { - if [module] =~ "osquery" { + if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja new file mode 100644 index 000000000..51e691176 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -0,0 +1,43 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} + +filter { + if [type] =~ "live_query" { + + mutate { + rename => { + "[host][hostname]" => "computer_name" + } + } + + prune { + blacklist_names => ["host"] + } + + split { + field => "rows" + } + } +} + + +output { + if [type] =~ "live_query" { + elasticsearch { + pipeline => "osquery.live_query" + hosts => "{{ ES }}" + index => "so-osquery" + template_name => "so-osquery" + template => "/templates/so-osquery-template.json" + template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} + } + } +} diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index b8dc5eb21..5f3a359b5 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -42,6 +42,7 @@ { "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.dataset:mysql | groupby mysql.command"}, { "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.dataset:notice | groupby notice.note notice.message"}, { "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.dataset:ntlm | groupby ntlm.server.dns.name"}, + { "name": "Osquery Live Queries", "description": "Osquery Live Query results grouped by computer name", "query": "event.dataset:live_query | groupby host.hostname"}, { "name": "PE", "description": "PE files list", "query": "event.dataset:pe | groupby file.machine file.os file.subsystem"}, { "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.dataset:radius | groupby user.name.keyword"}, { "name": "RDP", "description": "RDP grouped by client name", "query": "event.dataset:rdp | groupby client.name"},