From f613d8ad86467590ad16d9997c622513820f7a9f Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:36:18 +0000 Subject: [PATCH 1/8] Add RITA Logstash config --- pillar/logstash/search.sls | 1 + salt/filebeat/etc/filebeat.yml | 49 +++++++++++++++++++ .../config/so/9801_output_rita.conf.jinja | 22 +++++++++ 3 files changed, 72 insertions(+) create mode 100644 salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 917657e1f..ebe133056 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -13,4 +13,5 @@ logstash: - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja - so/9800_output_logscan.conf.jinja + - so/9801_output_rita.conf.jinja - so/9900_output_endgame.conf.jinja diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index e29b1a583..78151e28b 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -10,6 +10,7 @@ {%- set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %} {%- set WAZUHENABLED = salt['pillar.get']('global:wazuh', '0') %} {%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %} +{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) -%} {%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%} {%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%} {%- set FBMEMEVENTS = salt['pillar.get']('filebeat:mem_events', 2048) -%} @@ -264,6 +265,54 @@ filebeat.inputs: {%- endif %} +{%- if RITAENABLED %} +- type: filestream + paths: + - /nsm/rita/beacons.csv + exclude_lines: ['^Score', '^Source', '^Domain'] + fields: + module: rita + dataset: beacon + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + pipeline: "rita.beacon" + index: "so-rita" + +- type: filestream + paths: + - //nsm/rita/logs/long-connections.csv + - /nsm/rita/logs/open-connections.csv + exclude_lines: ['^Source'] + fields: + module: rita + dataset: connection + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + pipeline: "rita.connection" + index: "so-rita" + +- type: filestream + paths: + - /nsm/rita/exploded-dns.csv + exclude_lines: ['^Domain'] + fields: + module: rita + dataset: dns + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + pipeline: "rita.dns" + index: "so-rita" +{%- endif %} + {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %} - type: log paths: diff --git a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja new file mode 100644 index 000000000..40c6ad33c --- /dev/null +++ b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja @@ -0,0 +1,22 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +output { + if [module] =~ "rita" and "import" not in [tags] { + elasticsearch { + pipeline => "%{module}.%{dataset}" + hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" +{% endif %} + index => "so-rita" + ssl => true + ssl_certificate_verification => false + } + } +} From 2487d468aba09ca4c18d3fc67300093aaa8b327d Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:38:22 +0000 Subject: [PATCH 2/8] Add RITA Elasticsearch ingest pipeline config --- salt/elasticsearch/files/ingest/rita.beacon | 127 ++++++++++++++++++ .../files/ingest/rita.connection | 36 +++++ salt/elasticsearch/files/ingest/rita.dns | 39 ++++++ 3 files changed, 202 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/rita.beacon create mode 100644 salt/elasticsearch/files/ingest/rita.connection create mode 100644 salt/elasticsearch/files/ingest/rita.dns diff --git a/salt/elasticsearch/files/ingest/rita.beacon b/salt/elasticsearch/files/ingest/rita.beacon new file mode 100644 index 000000000..ab53be763 --- /dev/null +++ b/salt/elasticsearch/files/ingest/rita.beacon @@ -0,0 +1,127 @@ +{ + "description": "RITA Beacons", + "processors": [ + { + "set": { + "field": "_index", + "value": "so-rita", + "override": true + } + }, + { + "csv": { + "field": "message", + "target_fields": [ + "beacon.score", + "source.ip", + "destination.ip", + "network.connections", + "network.average_bytes", + "beacon.interval.range", + "beacon.size.range", + "beacon.interval.top", + "beacon.size.top", + "beacon.interval.top_count", + "beacon.size.top_count", + "beacon.interval.skew", + "beacon.size.skew", + "beacon.interval.dispersion", + "beacon.size.dispersion", + "network.bytes" + ] + } + }, + { + "convert": { + "field": "beacon.score", + "type": "float" + } + }, + { + "convert": { + "field": "network.connections", + "type": "integer" + } + }, + { + "convert": { + "field": "network.average_bytes", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.range", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.range", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.top", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.top", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.top_count", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.top_count", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.interval.skew", + "type": "float" + } + }, + { + "convert": { + "field": "beacon.size.skew", + "type": "float" + } + }, + { + "convert": { + "field": "beacon.interval.dispersion", + "type": "integer" + } + }, + { + "convert": { + "field": "beacon.size.dispersion", + "type": "integer" + } + }, + { + "convert": { + "field": "network.bytes", + "type": "integer" + } + }, + { "set": { "if": "ctx.beacon?.score == 1", "field": "dataset", "value": "alert", "override": true }}, + { "set": { "if": "ctx.beacon?.score == 1", "field": "rule.name", "value": "Potential C2 Beacon Activity", "override": true }}, + { "set": { "if": "ctx.beacon?.score == 1", "field": "event.severity", "value": 3, "override": true }}, + { + "pipeline": { + "name": "common" + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/rita.connection b/salt/elasticsearch/files/ingest/rita.connection new file mode 100644 index 000000000..58cc921f4 --- /dev/null +++ b/salt/elasticsearch/files/ingest/rita.connection @@ -0,0 +1,36 @@ +{ + "description": "RITA Connections", + "processors": [ + { + "set": { + "field": "_index", + "value": "so-rita", + "override": true + } + }, + { + "dissect": { + "field": "message", + "pattern": "%{source.ip},%{destination.ip},%{network.port}:%{network.protocol}:%{network.service},%{connection.duration},%{connection.state}" + } + }, + { + "convert": { + "field": "connection.duration", + "type": "float" + } + }, + { + "set": { + "field": "event.duration", + "value": "{{ connection.duration }}", + "override": true + } + }, + { + "pipeline": { + "name": "common" + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/rita.dns b/salt/elasticsearch/files/ingest/rita.dns new file mode 100644 index 000000000..7583bc320 --- /dev/null +++ b/salt/elasticsearch/files/ingest/rita.dns @@ -0,0 +1,39 @@ +{ + "description": "RITA DNS", + "processors": [ + { + "set": { + "field": "_index", + "value": "so-rita", + "override": true + } + }, + { + "csv": { + "field": "message", + "target_fields": [ + "dns.question.name", + "dns.question.subdomain_count", + "dns.question.count" + ] + } + }, + { + "convert": { + "field": "dns.question.subdomain_count", + "type": "integer" + } + }, + { + "convert": { + "field": "dns.question.count", + "type": "integer" + } + }, + { + "pipeline": { + "name": "common" + } + } + ] +} From 57f01c70ec422db2b2fabd4235a153c3fe11936b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:45:23 +0000 Subject: [PATCH 3/8] Remove extra forward slash in log path --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 78151e28b..243a2a031 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -283,7 +283,7 @@ filebeat.inputs: - type: filestream paths: - - //nsm/rita/logs/long-connections.csv + - /nsm/rita/logs/long-connections.csv - /nsm/rita/logs/open-connections.csv exclude_lines: ['^Source'] fields: From 8a56c8877305698171f956863f8c231f0adc5188 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 22 Mar 2022 17:51:17 +0000 Subject: [PATCH 4/8] Adjust log file paths --- salt/filebeat/etc/filebeat.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 243a2a031..7efa391e2 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -283,8 +283,8 @@ filebeat.inputs: - type: filestream paths: - - /nsm/rita/logs/long-connections.csv - - /nsm/rita/logs/open-connections.csv + - /nsm/rita/long-connections.csv + - /nsm/rita/open-connections.csv exclude_lines: ['^Source'] fields: module: rita From 1f2bca599fe044b5e6d719d625d3f3c7e12ed343 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 23 Mar 2022 11:00:26 -0400 Subject: [PATCH 5/8] Check cluster health before trying to load roles for ES --- salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load index c407ac183..7ce907f87 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load @@ -33,6 +33,8 @@ while [[ "$COUNT" -le 240 ]]; do if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" + # Check cluster health once connected + so-elasticsearch-query _cluster/health?wait_for_status=yellow > /dev/null 2>&1 break else ((COUNT+=1)) From fe1b72655bdae9b84ce67814d3dd3ce3085f7fe7 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 24 Mar 2022 16:45:06 +0000 Subject: [PATCH 6/8] Additional .keyword shims for process mappings --- .../component/so/dtc-process-mappings.json | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/salt/elasticsearch/templates/component/so/dtc-process-mappings.json b/salt/elasticsearch/templates/component/so/dtc-process-mappings.json index 2b8d8abfb..d3d22139a 100644 --- a/salt/elasticsearch/templates/component/so/dtc-process-mappings.json +++ b/salt/elasticsearch/templates/component/so/dtc-process-mappings.json @@ -60,6 +60,32 @@ }, "type": "wildcard" }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + }, + "executable": { + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, "name": { "fields": { "keyword": { @@ -73,6 +99,133 @@ "ignore_above": 1024, "type": "keyword" }, + "parent": { + "properties": { + "command_line": { + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "text": { + "type": "match_only_text" + }, + "keyword": { + "type": "keyword" + } + }, + "type": "wildcard" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + }, + "executable": { + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + }, + "company": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + }, + "description": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + } + } + } + }, "pid": { "type": "long", "fields": { @@ -88,6 +241,19 @@ "type": "keyword" } } + }, + "working_directory": { + "fields": { + "security": { + "type": "text", + "analyzer": "es_security_analyzer" + }, + "keyword": { + "type": "keyword" + } + }, + "ignore_above": 1024, + "type": "keyword" } } } From fbc86f43ec9e13e2800a0a52997ba85b389ad459 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 24 Mar 2022 13:03:03 -0400 Subject: [PATCH 7/8] Add exclude filter for logs for when there are no results from analysis --- salt/filebeat/etc/filebeat.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 7efa391e2..62a45e9c4 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -269,7 +269,7 @@ filebeat.inputs: - type: filestream paths: - /nsm/rita/beacons.csv - exclude_lines: ['^Score', '^Source', '^Domain'] + exclude_lines: ['^Score', '^Source', '^Domain', '^No results'] fields: module: rita dataset: beacon @@ -285,7 +285,7 @@ filebeat.inputs: paths: - /nsm/rita/long-connections.csv - /nsm/rita/open-connections.csv - exclude_lines: ['^Source'] + exclude_lines: ['^Source', '^No results'] fields: module: rita dataset: connection @@ -300,7 +300,7 @@ filebeat.inputs: - type: filestream paths: - /nsm/rita/exploded-dns.csv - exclude_lines: ['^Domain'] + exclude_lines: ['^Domain', '^No results'] fields: module: rita dataset: dns From e6599cd10eb62c06e0e56506fe39f0642cddfff4 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 25 Mar 2022 13:57:44 -0400 Subject: [PATCH 8/8] Update with changes from Abe's PR and other fixes --- salt/elasticsearch/files/ingest/syslog | 177 +++++++++++++++++++++---- 1 file changed, 149 insertions(+), 28 deletions(-) diff --git a/salt/elasticsearch/files/ingest/syslog b/salt/elasticsearch/files/ingest/syslog index bf40efec5..8919c3e1f 100644 --- a/salt/elasticsearch/files/ingest/syslog +++ b/salt/elasticsearch/files/ingest/syslog @@ -1,36 +1,157 @@ { - "description" : "syslog", + "description" : "syslog pipeline", "processors" : [ { - "dissect": { - "field": "message", - "pattern" : "%{message}", - "on_failure": [ { "drop" : { } } ] - }, - "remove": { - "field": [ "type", "agent" ], - "ignore_failure": true - } + "dissect": { + "field": "message", + "pattern" : "%{message}", + "on_failure": [ { "drop" : { } } ] + }, + "remove": { + "field": [ "type", "agent" ], + "ignore_failure": true + } + }, { + "grok": { + "field": "message", + "patterns": [ + "^<%{INT:syslog.priority:int}>%{TIMESTAMP_ISO8601:syslog.timestamp} +%{IPORHOST:syslog.host} +%{PROG:syslog.program}(?:\\[%{POSINT:syslog.pid:int}\\])?: %{GREEDYDATA:real_message}$", + + "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$", + + "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$" + ], + "ignore_failure": true + } }, { - "grok": - { - "field": "message", - "patterns": [ - "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$", - "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$" - ], - "ignore_failure": true - } + "convert" : { + "if": "ctx?.syslog?.priority != null", + "field" : "syslog.priority", + "type": "integer" + } }, - { "set": { "if": "ctx.source?.application == 'filterlog'", "field": "dataset", "value": "firewall", "ignore_failure": true } }, - { "set": { "if": "ctx.vendor != null", "field": "module", "value": "{{ vendor }}", "ignore_failure": true } }, - { "set": { "if": "ctx.product != null", "field": "dataset", "value": "{{ product }}", "ignore_failure": true } }, - { "set": { "field": "event.ingested", "value": "{{ @timestamp }}" } }, - { "date": { "if": "ctx.syslog?.timestamp != null", "field": "syslog.timestamp", "target_field": "@timestamp", "formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], "ignore_failure": true } }, - { "remove": { "field": ["pid", "program"], "ignore_missing": true, "ignore_failure": true } }, - { "pipeline": { "if": "ctx.vendor != null && ctx.product != null", "name": "{{ vendor }}.{{ product }}", "ignore_failure": true } }, - { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog", "ignore_failure": true } }, - { "pipeline": { "name": "common" } } + { + "script": { + "description": "Map syslog priority into facility and level", + "lang": "painless", + "params" : { + "level": [ + "emerg", + "alert", + "crit", + "err", + "warn", + "notice", + "info", + "debug" + ], + "facility" : [ + "kern", + "user", + "mail", + "daemon", + "auth", + "syslog", + "lpr", + "news", + "uucp", + "cron", + "authpriv", + "ftp", + "ntp", + "security", + "console", + "solaris-cron", + "local0", + "local1", + "local2", + "local3", + "local4", + "local5", + "local6", + "local7" + ] + }, + "source": "if (ctx['syslog'] != null && ctx['syslog']['priority'] != null) { int p = ctx['syslog']['priority']; int f = p / 8; int l = p - (f * 8); ctx['syslog']['facility_label'] = [ : ]; ctx['syslog']['severity_label'] = [ : ]; ctx['syslog'].put('severity', l); ctx['syslog'].put('severity_label', params.level[l].toUpperCase()); ctx['syslog'].put('facility', f); ctx['syslog'].put('facility_label', params.facility[f].toUpperCase()); }" + + } + }, + { + "set": { + "if": "ctx.syslog?.host != null", + "field": "host.name", + "value": "{{ syslog.host }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.syslog?.program != null", + "field": "process.name", + "value": "{{ syslog.program }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.syslog?.pid != null", + "field": "process.id", + "value": "{{ syslog.pid }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.source?.application == 'filterlog'", + "field": "dataset", + "value": "firewall", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.vendor != null", + "field": "module", + "value": "{{ vendor }}", + "ignore_failure": true + } + }, { + "set": { + "if": "ctx.product != null", + "field": "dataset", + "value": "{{ product }}", + "ignore_failure": true + } + }, { + "set": { + "field": "ingest.timestamp", + "value": "{{ @timestamp }}" + } + }, { + "date": { + "if": "ctx.syslog?.timestamp != null", + "field": "syslog.timestamp", + "target_field": "@timestamp", + "formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], + "ignore_failure": true + } + }, { + "remove": { + "field": ["pid", "program"], + "ignore_missing": true, + "ignore_failure": true + } + }, { + "pipeline": { + "if": "ctx.vendor != null && ctx.product != null", + "name": "{{ vendor }}.{{ product }}", + "ignore_failure": true + } + }, { + "pipeline": { + "if": "ctx.dataset == 'firewall'", + "name": "filterlog", + "ignore_failure": true + } + }, { + "pipeline": { "name": "common" } + } ] }