CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-07 09:42:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3339 from Security-Onion-Solutions/feature/live_query-hunt

Browse Source 
Feature/live query hunt
This commit is contained in:
Josh Brower
2021-03-08 08:31:25 -05:00
committed by GitHub
parent dc673eef77 548f67ca6f
commit 19b3c7bb07
10 changed files with 106 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pillar/logstash/manager.sls
View File

@@ -3,6 +3,7 @@ logstash:
pipelines:
manager:
config:
- so/0008_input_fleet_livequery.conf.jinja
- so/0009_input_beats.conf
- so/0010_input_hhbeats.conf
- so/9999_output_redis.conf.jinja

1
pillar/logstash/search.sls
View File

@@ -8,6 +8,7 @@ logstash:
- so/9002_output_import.conf.jinja
- so/9034_output_syslog.conf.jinja
- so/9100_output_osquery.conf.jinja
- so/9101_output_osquery_livequery.conf.jinja
- so/9400_output_suricata.conf.jinja
- so/9500_output_beats.conf.jinja
- so/9600_output_ossec.conf.jinja

16
salt/elasticsearch/files/ingest/osquery.live_query Normal file
View File

@@ -0,0 +1,16 @@
{
"description" : "osquery live query",
"processors" : [
{
"script": {
"lang": "painless",
"source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; "
}
},
{ "remove": { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } },
{ "rename": { "field": "distributed_query_execution_id", "target_field": "result.query_id", "ignore_missing": true } },
{ "rename": { "field": "computer_name", "target_field": "host.hostname", "ignore_missing": true } },
{ "pipeline": { "name": "osquery.normalize" } },
{ "pipeline": { "name": "common" } }
]
}

14
salt/elasticsearch/files/ingest/osquery.normalize Normal file
View File

@@ -0,0 +1,14 @@
{
"description" : "osquery normalize",
"processors" : [
{ "rename": { "field": "result.columns.cmdline", "target_field": "process.command_line", "ignore_missing": true } },
{ "rename": { "field": "result.columns.cwd", "target_field": "process.working_directory", "ignore_missing": true } },
{ "rename": { "field": "result.columns.name", "target_field": "process.name", "ignore_missing": true } },
{ "rename": { "field": "result.columns.path", "target_field": "process.executable", "ignore_missing": true } },
{ "rename": { "field": "result.columns.pid", "target_field": "process.pid", "ignore_missing": true } },
{ "rename": { "field": "result.columns.parent", "target_field": "process.ppid", "ignore_missing": true } },
{ "rename": { "field": "result.columns.uid", "target_field": "user.id", "ignore_missing": true } },
{ "rename": { "field": "result.columns.username", "target_field": "user.name", "ignore_missing": true } },
{ "rename": { "field": "result.columns.gid", "target_field": "group.id", "ignore_missing": true } }
]
}

17
salt/elasticsearch/files/ingest/osquery.query_result
View File

@@ -1,24 +1,19 @@
{
"description" : "osquery",
"processors" : [
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "gsub": { "field": "message2.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } },
{ "rename": { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns", "target_field": "winlog", "ignore_missing": true } },
{ "json": { "field": "message", "target_field": "result", "ignore_failure": true } },
{ "gsub": { "field": "result.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } },
{ "rename": { "if": "ctx.result.columns?.eventid != null", "field": "result.columns", "target_field": "winlog", "ignore_missing": true } },
{ "json": { "field": "winlog.data", "target_field": "unparsed", "ignore_failure": true} },
{ "set": { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true } },
{ "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } },
{ "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } },
{ "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } },
{ "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } },
{ "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } },
{
"script": {
"lang": "painless",
"source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; "
}
},
{ "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } },
{ "set": { "field": "event.module", "value": "osquery", "override": false } },
{ "set": { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false} },
{ "set": { "field": "event.dataset", "value": "{{result.name}}", "override": false} },
{ "pipeline": { "if": "!(ctx.containsKey('winlog'))", "name": "osquery.normalize" } },
{ "pipeline": { "name": "common" } }
]
}

4
salt/elasticsearch/templates/so/so-common-template.json
View File

@@ -365,6 +365,10 @@
"request":{
"type":"object",
"dynamic": true
},
"result":{
"type":"object",
"dynamic": true
},
"rfb":{
"type":"object",

19
salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja Normal file
View File

@@ -0,0 +1,19 @@
{%- set MANAGER = salt['grains.get']('master') %}
{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
input {
redis {
host => '{{ MANAGER }}'
port => 6379
data_type => 'pattern_channel'
key => 'results_*'
type => 'live_query'
add_field => {
"module" => "osquery"
"dataset" => "live_query"
}
threads => {{ THREADS }}
batch_count => {{ BATCH }}
}
}

2
salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
View File

@@ -5,7 +5,7 @@
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "osquery" {
if [module] =~ "osquery" and "live_query" not in [dataset] {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"

43
salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja Normal file
View File

@@ -0,0 +1,43 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
filter {
if [type] =~ "live_query" {
mutate {
rename => {
"[host][hostname]" => "computer_name"
}
}
prune {
blacklist_names => ["host"]
}
split {
field => "rows"
}
}
}
output {
if [type] =~ "live_query" {
elasticsearch {
pipeline => "osquery.live_query"
hosts => "{{ ES }}"
index => "so-osquery"
template_name => "so-osquery"
template => "/templates/so-osquery-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

1
salt/soc/files/soc/hunt.queries.json
View File

@@ -42,6 +42,7 @@
{ "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.dataset:mysql | groupby mysql.command"},
{ "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.dataset:notice | groupby notice.note notice.message"},
{ "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.dataset:ntlm | groupby ntlm.server.dns.name"},
{ "name": "Osquery Live Queries", "description": "Osquery Live Query results grouped by computer name", "query": "event.dataset:live_query | groupby host.hostname"},
{ "name": "PE", "description": "PE files list", "query": "event.dataset:pe | groupby file.machine file.os file.subsystem"},
{ "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.dataset:radius | groupby user.name.keyword"},
{ "name": "RDP", "description": "RDP grouped by client name", "query": "event.dataset:rdp | groupby client.name"},