From f613d8ad86467590ad16d9997c622513820f7a9f Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:36:18 +0000 Subject: [PATCH 1/5] Add RITA Logstash config --- pillar/logstash/search.sls | 1 + salt/filebeat/etc/filebeat.yml | 49 +++++++++++++++++++ .../config/so/9801_output_rita.conf.jinja | 22 +++++++++ 3 files changed, 72 insertions(+) create mode 100644 salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 917657e1f..ebe133056 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -13,4 +13,5 @@ logstash: - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja - so/9800_output_logscan.conf.jinja + - so/9801_output_rita.conf.jinja - so/9900_output_endgame.conf.jinja diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index e29b1a583..78151e28b 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -10,6 +10,7 @@ {%- set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %} {%- set WAZUHENABLED = salt['pillar.get']('global:wazuh', '0') %} {%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %} +{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) -%} {%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%} {%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%} {%- set FBMEMEVENTS = salt['pillar.get']('filebeat:mem_events', 2048) -%} @@ -264,6 +265,54 @@ filebeat.inputs: {%- endif %} +{%- if RITAENABLED %} +- type: filestream + paths: + - /nsm/rita/beacons.csv + exclude_lines: ['^Score', '^Source', '^Domain'] + fields: + module: rita + dataset: beacon + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + pipeline: "rita.beacon" + index: "so-rita" + +- type: filestream + paths: + - //nsm/rita/logs/long-connections.csv + - /nsm/rita/logs/open-connections.csv + exclude_lines: ['^Source'] + fields: + module: rita + dataset: connection + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + pipeline: "rita.connection" + index: "so-rita" + +- type: filestream + paths: + - /nsm/rita/exploded-dns.csv + exclude_lines: ['^Domain'] + fields: + module: rita + dataset: dns + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + pipeline: "rita.dns" + index: "so-rita" +{%- endif %} + {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %} - type: log paths: diff --git a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja new file mode 100644 index 000000000..40c6ad33c --- /dev/null +++ b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja @@ -0,0 +1,22 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +output { + if [module] =~ "rita" and "import" not in [tags] { + elasticsearch { + pipeline => "%{module}.%{dataset}" + hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" +{% endif %} + index => "so-rita" + ssl => true + ssl_certificate_verification => false + } + } +} From 2487d468aba09ca4c18d3fc67300093aaa8b327d Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:38:22 +0000 Subject: [PATCH 2/5] Add RITA Elasticsearch ingest pipeline config --- salt/elasticsearch/files/ingest/rita.beacon | 127 ++++++++++++++++++ .../files/ingest/rita.connection | 36 +++++ salt/elasticsearch/files/ingest/rita.dns | 39 ++++++ 3 files changed, 202 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/rita.beacon create mode 100644 salt/elasticsearch/files/ingest/rita.connection create mode 100644 salt/elasticsearch/files/ingest/rita.dns diff --git a/salt/elasticsearch/files/ingest/rita.beacon b/salt/elasticsearch/files/ingest/rita.beacon new file mode 100644 index 000000000..ab53be763 --- /dev/null +++ b/salt/elasticsearch/files/ingest/rita.beacon @@ -0,0 +1,127 @@ +{ + "description": "RITA Beacons", + "processors": [ + { + "set": { + "field": "_index", + "value": "so-rita", + "override": true + } + }, + { + "csv": { + "field": "message", + "target_fields": [ + "beacon.score", + "source.ip", + "destination.ip", + "network.connections", + "network.average_bytes", + "beacon.interval.range", + "beacon.size.range", + "beacon.interval.top", + "beacon.size.top", + "beacon.interval.top_count", + "beacon.size.top_count", + "beacon.interval.skew", + "beacon.size.skew", + "beacon.interval.dispersion", + "beacon.size.dispersion", + "network.bytes" + ] + } + }, + { + "convert": { + "field": "beacon.score", + "type": "float" + } + }, + { + "convert": { + "field": "network.connections", + "type": "integer" + } + }, + { + "convert": { + "field": "network.average_bytes", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.range", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.range", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.top", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.top", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.top_count", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.top_count", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.skew", + "type": "float" + } + }, + { + "convert": { + "field": "beacon.size.skew", + "type": "float" + } + }, + { + "convert": { + "field": "beacon.interval.dispersion", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.dispersion", + "type": "integer" + } + }, + { + "convert": { + "field": "network.bytes", + "type": "integer" + } + }, + { "set": { "if": "ctx.beacon?.score == 1", "field": "dataset", "value": "alert", "override": true }}, + { "set": { "if": "ctx.beacon?.score == 1", "field": "rule.name", "value": "Potential C2 Beacon Activity", "override": true }}, + { "set": { "if": "ctx.beacon?.score == 1", "field": "event.severity", "value": 3, "override": true }}, + { + "pipeline": { + "name": "common" + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/rita.connection b/salt/elasticsearch/files/ingest/rita.connection new file mode 100644 index 000000000..58cc921f4 --- /dev/null +++ b/salt/elasticsearch/files/ingest/rita.connection @@ -0,0 +1,36 @@ +{ + "description": "RITA Connections", + "processors": [ + { + "set": { + "field": "_index", + "value": "so-rita", + "override": true + } + }, + { + "dissect": { + "field": "message", + "pattern": "%{source.ip},%{destination.ip},%{network.port}:%{network.protocol}:%{network.service},%{connection.duration},%{connection.state}" + } + }, + { + "convert": { + "field": "connection.duration", + "type": "float" + } + }, + { + "set": { + "field": "event.duration", + "value": "{{ connection.duration }}", + "override": true + } + }, + { + "pipeline": { + "name": "common" + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/rita.dns b/salt/elasticsearch/files/ingest/rita.dns new file mode 100644 index 000000000..7583bc320 --- /dev/null +++ b/salt/elasticsearch/files/ingest/rita.dns @@ -0,0 +1,39 @@ +{ + "description": "RITA DNS", + "processors": [ + { + "set": { + "field": "_index", + "value": "so-rita", + "override": true + } + }, + { + "csv": { + "field": "message", + "target_fields": [ + "dns.question.name", + "dns.question.subdomain_count", + "dns.question.count" + ] + } + }, + { + "convert": { + "field": "dns.question.subdomain_count", + "type": "integer" + } + }, + { + "convert": { + "field": "dns.question.count", + "type": "integer" + } + }, + { + "pipeline": { + "name": "common" + } + } + ] +} From 57f01c70ec422db2b2fabd4235a153c3fe11936b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:45:23 +0000 Subject: [PATCH 3/5] Remove extra forward slash in log path --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 78151e28b..243a2a031 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -283,7 +283,7 @@ filebeat.inputs: - type: filestream paths: - - //nsm/rita/logs/long-connections.csv + - /nsm/rita/logs/long-connections.csv - /nsm/rita/logs/open-connections.csv exclude_lines: ['^Source'] fields: From 8a56c8877305698171f956863f8c231f0adc5188 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:51:17 +0000 Subject: [PATCH 4/5] Adjust log file paths --- salt/filebeat/etc/filebeat.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 243a2a031..7efa391e2 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -283,8 +283,8 @@ filebeat.inputs: - type: filestream paths: - - /nsm/rita/logs/long-connections.csv - - /nsm/rita/logs/open-connections.csv + - /nsm/rita/long-connections.csv + - /nsm/rita/open-connections.csv exclude_lines: ['^Source'] fields: module: rita From fbc86f43ec9e13e2800a0a52997ba85b389ad459 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 24 Mar 2022 13:03:03 -0400 Subject: [PATCH 5/5] Add exclude filter for logs for when there are no results from analysis --- salt/filebeat/etc/filebeat.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 7efa391e2..62a45e9c4 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -269,7 +269,7 @@ filebeat.inputs: - type: filestream paths: - /nsm/rita/beacons.csv - exclude_lines: ['^Score', '^Source', '^Domain'] + exclude_lines: ['^Score', '^Source', '^Domain', '^No results'] fields: module: rita dataset: beacon @@ -285,7 +285,7 @@ filebeat.inputs: paths: - /nsm/rita/long-connections.csv - /nsm/rita/open-connections.csv - exclude_lines: ['^Source'] + exclude_lines: ['^Source', '^No results'] fields: module: rita dataset: connection @@ -300,7 +300,7 @@ filebeat.inputs: - type: filestream paths: - /nsm/rita/exploded-dns.csv - exclude_lines: ['^Domain'] + exclude_lines: ['^Domain', '^No results'] fields: module: rita dataset: dns