CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-01-24 00:43:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into kilo

Browse Source
This commit is contained in:
Jason Ertel
2022-03-28 15:53:33 -04:00
parent deb9b0e5ef fb7160cba5
commit c23b87965f
9 changed files with 591 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pillar/logstash/search.sls
View File

@@ -13,4 +13,5 @@ logstash:
- so/9600_output_ossec.conf.jinja
- so/9700_output_strelka.conf.jinja
- so/9800_output_logscan.conf.jinja
- so/9801_output_rita.conf.jinja
- so/9900_output_endgame.conf.jinja

127
salt/elasticsearch/files/ingest/rita.beacon Normal file
View File

@@ -0,0 +1,127 @@
{
"description": "RITA Beacons",
"processors": [
{
"set": {
"field": "_index",
"value": "so-rita",
"override": true
}
},
{
"csv": {
"field": "message",
"target_fields": [
"beacon.score",
"source.ip",
"destination.ip",
"network.connections",
"network.average_bytes",
"beacon.interval.range",
"beacon.size.range",
"beacon.interval.top",
"beacon.size.top",
"beacon.interval.top_count",
"beacon.size.top_count",
"beacon.interval.skew",
"beacon.size.skew",
"beacon.interval.dispersion",
"beacon.size.dispersion",
"network.bytes"
]
}
},
{
"convert": {
"field": "beacon.score",
"type": "float"
}
},
{
"convert": {
"field": "network.connections",
"type": "integer"
}
},
{
"convert": {
"field": "network.average_bytes",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.interval.range",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.size.range",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.interval.top",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.size.top",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.interval.top_count",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.size.top_count",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.interval.skew",
"type": "float"
}
},
{
"convert": {
"field": "beacon.size.skew",
"type": "float"
}
},
{
"convert": {
"field": "beacon.interval.dispersion",
"type": "integer"
}
},
{
"convert": {
"field": "beacon.size.dispersion",
"type": "integer"
}
},
{
"convert": {
"field": "network.bytes",
"type": "integer"
}
},
{ "set": { "if": "ctx.beacon?.score == 1", "field": "dataset", "value": "alert", "override": true }},
{ "set": { "if": "ctx.beacon?.score == 1", "field": "rule.name", "value": "Potential C2 Beacon Activity", "override": true }},
{ "set": { "if": "ctx.beacon?.score == 1", "field": "event.severity", "value": 3, "override": true }},
{
"pipeline": {
"name": "common"
}
}
]
}

36
salt/elasticsearch/files/ingest/rita.connection Normal file
View File

@@ -0,0 +1,36 @@
{
"description": "RITA Connections",
"processors": [
{
"set": {
"field": "_index",
"value": "so-rita",
"override": true
}
},
{
"dissect": {
"field": "message",
"pattern": "%{source.ip},%{destination.ip},%{network.port}:%{network.protocol}:%{network.service},%{connection.duration},%{connection.state}"
}
},
{
"convert": {
"field": "connection.duration",
"type": "float"
}
},
{
"set": {
"field": "event.duration",
"value": "{{ connection.duration }}",
"override": true
}
},
{
"pipeline": {
"name": "common"
}
}
]
}

39
salt/elasticsearch/files/ingest/rita.dns Normal file
View File

@@ -0,0 +1,39 @@
{
"description": "RITA DNS",
"processors": [
{
"set": {
"field": "_index",
"value": "so-rita",
"override": true
}
},
{
"csv": {
"field": "message",
"target_fields": [
"dns.question.name",
"dns.question.subdomain_count",
"dns.question.count"
]
}
},
{
"convert": {
"field": "dns.question.subdomain_count",
"type": "integer"
}
},
{
"convert": {
"field": "dns.question.count",
"type": "integer"
}
},
{
"pipeline": {
"name": "common"
}
}
]
}

177
salt/elasticsearch/files/ingest/syslog
View File

@@ -1,36 +1,157 @@
{
"description" : "syslog",
"description" : "syslog pipeline",
"processors" : [
{
"dissect": {
"field": "message",
"pattern" : "%{message}",
"on_failure": [ { "drop" : { } } ]
},
"remove": {
"field": [ "type", "agent" ],
"ignore_failure": true
}
"dissect": {
"field": "message",
"pattern" : "%{message}",
"on_failure": [ { "drop" : { } } ]
},
"remove": {
"field": [ "type", "agent" ],
"ignore_failure": true
}
}, {
"grok": {
"field": "message",
"patterns": [
"^<%{INT:syslog.priority:int}>%{TIMESTAMP_ISO8601:syslog.timestamp} +%{IPORHOST:syslog.host} +%{PROG:syslog.program}(?:\\[%{POSINT:syslog.pid:int}\\])?: %{GREEDYDATA:real_message}$",
"^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$",
"^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
],
"ignore_failure": true
}
},
{
"grok":
{
"field": "message",
"patterns": [
"^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$",
"^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
],
"ignore_failure": true
}
"convert" : {
"if": "ctx?.syslog?.priority != null",
"field" : "syslog.priority",
"type": "integer"
}
},
{ "set": { "if": "ctx.source?.application == 'filterlog'", "field": "dataset", "value": "firewall", "ignore_failure": true } },
{ "set": { "if": "ctx.vendor != null", "field": "module", "value": "{{ vendor }}", "ignore_failure": true } },
{ "set": { "if": "ctx.product != null", "field": "dataset", "value": "{{ product }}", "ignore_failure": true } },
{ "set": { "field": "event.ingested", "value": "{{ @timestamp }}" } },
{ "date": { "if": "ctx.syslog?.timestamp != null", "field": "syslog.timestamp", "target_field": "@timestamp", "formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], "ignore_failure": true } },
{ "remove": { "field": ["pid", "program"], "ignore_missing": true, "ignore_failure": true } },
{ "pipeline": { "if": "ctx.vendor != null && ctx.product != null", "name": "{{ vendor }}.{{ product }}", "ignore_failure": true } },
{ "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog", "ignore_failure": true } },
{ "pipeline": { "name": "common" } }
{
"script": {
"description": "Map syslog priority into facility and level",
"lang": "painless",
"params" : {
"level": [
"emerg",
"alert",
"crit",
"err",
"warn",
"notice",
"info",
"debug"
],
"facility" : [
"kern",
"user",
"mail",
"daemon",
"auth",
"syslog",
"lpr",
"news",
"uucp",
"cron",
"authpriv",
"ftp",
"ntp",
"security",
"console",
"solaris-cron",
"local0",
"local1",
"local2",
"local3",
"local4",
"local5",
"local6",
"local7"
]
},
"source": "if (ctx['syslog'] != null && ctx['syslog']['priority'] != null) { int p = ctx['syslog']['priority']; int f = p / 8; int l = p - (f * 8); ctx['syslog']['facility_label'] = [ : ]; ctx['syslog']['severity_label'] = [ : ]; ctx['syslog'].put('severity', l); ctx['syslog'].put('severity_label', params.level[l].toUpperCase()); ctx['syslog'].put('facility', f); ctx['syslog'].put('facility_label', params.facility[f].toUpperCase()); }"
}
},
{
"set": {
"if": "ctx.syslog?.host != null",
"field": "host.name",
"value": "{{ syslog.host }}",
"ignore_failure": true
}
}, {
"set": {
"if": "ctx.syslog?.program != null",
"field": "process.name",
"value": "{{ syslog.program }}",
"ignore_failure": true
}
}, {
"set": {
"if": "ctx.syslog?.pid != null",
"field": "process.id",
"value": "{{ syslog.pid }}",
"ignore_failure": true
}
}, {
"set": {
"if": "ctx.source?.application == 'filterlog'",
"field": "dataset",
"value": "firewall",
"ignore_failure": true
}
}, {
"set": {
"if": "ctx.vendor != null",
"field": "module",
"value": "{{ vendor }}",
"ignore_failure": true
}
}, {
"set": {
"if": "ctx.product != null",
"field": "dataset",
"value": "{{ product }}",
"ignore_failure": true
}
}, {
"set": {
"field": "ingest.timestamp",
"value": "{{ @timestamp }}"
}
}, {
"date": {
"if": "ctx.syslog?.timestamp != null",
"field": "syslog.timestamp",
"target_field": "@timestamp",
"formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"],
"ignore_failure": true
}
}, {
"remove": {
"field": ["pid", "program"],
"ignore_missing": true,
"ignore_failure": true
}
}, {
"pipeline": {
"if": "ctx.vendor != null && ctx.product != null",
"name": "{{ vendor }}.{{ product }}",
"ignore_failure": true
}
}, {
"pipeline": {
"if": "ctx.dataset == 'firewall'",
"name": "filterlog",
"ignore_failure": true
}
}, {
"pipeline": { "name": "common" }
}
]
}

166
salt/elasticsearch/templates/component/so/dtc-process-mappings.json
View File

@@ -60,6 +60,32 @@
},
"type": "wildcard"
},
"entity_id": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
},
"executable": {
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"fields": {
"keyword": {
@@ -73,6 +99,133 @@
"ignore_above": 1024,
"type": "keyword"
},
"parent": {
"properties": {
"command_line": {
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"text": {
"type": "match_only_text"
},
"keyword": {
"type": "keyword"
}
},
"type": "wildcard"
},
"entity_id": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
},
"executable": {
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
},
"ignore_above": 1024,
"type": "keyword"
}
}
},
"pe": {
"properties": {
"architecture": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
},
"company": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
},
"description": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
},
"file_version": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
},
"original_file_name": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
},
"product": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
}
}
}
},
"pid": {
"type": "long",
"fields": {
@@ -88,6 +241,19 @@
"type": "keyword"
}
}
},
"working_directory": {
"fields": {
"security": {
"type": "text",
"analyzer": "es_security_analyzer"
},
"keyword": {
"type": "keyword"
}
},
"ignore_above": 1024,
"type": "keyword"
}
}
}

2
salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load
View File

@@ -33,6 +33,8 @@ while [[ "$COUNT" -le 240 ]]; do
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
# Check cluster health once connected
so-elasticsearch-query _cluster/health?wait_for_status=yellow > /dev/null 2>&1
break
else
((COUNT+=1))

49
salt/filebeat/etc/filebeat.yml
View File

@@ -10,6 +10,7 @@
{%- set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %}
{%- set WAZUHENABLED = salt['pillar.get']('global:wazuh', '0') %}
{%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %}
{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) -%}
{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
{%- set FBMEMEVENTS = salt['pillar.get']('filebeat:mem_events', 2048) -%}
@@ -264,6 +265,54 @@ filebeat.inputs:
{%- endif %}
{%- if RITAENABLED %}
- type: filestream
paths:
- /nsm/rita/beacons.csv
exclude_lines: ['^Score', '^Source', '^Domain', '^No results']
fields:
module: rita
dataset: beacon
category: network
processors:
- drop_fields:
fields: ["source", "prospector", "input", "offset", "beat"]
fields_under_root: true
pipeline: "rita.beacon"
index: "so-rita"
- type: filestream
paths:
- /nsm/rita/long-connections.csv
- /nsm/rita/open-connections.csv
exclude_lines: ['^Source', '^No results']
fields:
module: rita
dataset: connection
category: network
processors:
- drop_fields:
fields: ["source", "prospector", "input", "offset", "beat"]
fields_under_root: true
pipeline: "rita.connection"
index: "so-rita"
- type: filestream
paths:
- /nsm/rita/exploded-dns.csv
exclude_lines: ['^Domain', '^No results']
fields:
module: rita
dataset: dns
category: network
processors:
- drop_fields:
fields: ["source", "prospector", "input", "offset", "beat"]
fields_under_root: true
pipeline: "rita.dns"
index: "so-rita"
{%- endif %}
{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %}
- type: log
paths:

22
salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja Normal file
View File

@@ -0,0 +1,22 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
output {
if [module] =~ "rita" and "import" not in [tags] {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
user => "{{ ES_USER }}"
password => "{{ ES_PASS }}"
{% endif %}
index => "so-rita"
ssl => true
ssl_certificate_verification => false
}
}
}