mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 09:12:45 +01:00
Merge branch 'dev' of github.com:Security-Onion-Solutions/securityonion-saltstack into dev
This commit is contained in:
William Wernert
1
pillar/data/mastersearchtab.sls
Normal file
1
pillar/data/mastersearchtab.sls
Normal file
@@ -0,0 +1 @@
|
||||
mastersearchtab:
|
||||
@@ -1,107 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# HiveAlerter modified from original at: https://raw.githubusercontent.com/Nclose-ZA/elastalert_hive_alerter/master/elastalert_hive_alerter/hive_alerter.py
|
||||
|
||||
import uuid
|
||||
|
||||
from elastalert.alerts import Alerter
|
||||
from thehive4py.api import TheHiveApi
|
||||
from thehive4py.models import Alert, AlertArtifact, CustomFieldHelper
|
||||
|
||||
|
||||
class TheHiveAlerter(Alerter):
|
||||
"""
|
||||
Use matched data to create alerts containing observables in an instance of TheHive
|
||||
"""
|
||||
|
||||
required_options = set(['hive_connection', 'hive_alert_config'])
|
||||
|
||||
def get_aggregation_summary_text(self, matches):
|
||||
text = super(TheHiveAlerter, self).get_aggregation_summary_text(matches)
|
||||
if text:
|
||||
text = '```\n{0}```\n'.format(text)
|
||||
return text
|
||||
|
||||
def create_artifacts(self, match):
|
||||
artifacts = []
|
||||
context = {'rule': self.rule, 'match': match}
|
||||
for mapping in self.rule.get('hive_observable_data_mapping', []):
|
||||
for observable_type, match_data_key in mapping.items():
|
||||
try:
|
||||
artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context)))
|
||||
except KeyError as e:
|
||||
print(('format string {} fail cause no key {} in {}'.format(e, match_data_key, context)))
|
||||
return artifacts
|
||||
|
||||
def create_alert_config(self, match):
|
||||
context = {'rule': self.rule, 'match': match}
|
||||
alert_config = {
|
||||
'artifacts': self.create_artifacts(match),
|
||||
'sourceRef': str(uuid.uuid4())[0:6],
|
||||
'title': '{rule[name]}'.format(**context)
|
||||
}
|
||||
|
||||
alert_config.update(self.rule.get('hive_alert_config', {}))
|
||||
|
||||
for alert_config_field, alert_config_value in alert_config.items():
|
||||
if alert_config_field == 'customFields':
|
||||
custom_fields = CustomFieldHelper()
|
||||
for cf_key, cf_value in alert_config_value.items():
|
||||
try:
|
||||
func = getattr(custom_fields, 'add_{}'.format(cf_value['type']))
|
||||
except AttributeError:
|
||||
raise Exception('unsupported custom field type {}'.format(cf_value['type']))
|
||||
value = cf_value['value'].format(**context)
|
||||
func(cf_key, value)
|
||||
alert_config[alert_config_field] = custom_fields.build()
|
||||
elif isinstance(alert_config_value, str):
|
||||
alert_config[alert_config_field] = alert_config_value.format(**context)
|
||||
elif isinstance(alert_config_value, (list, tuple)):
|
||||
formatted_list = []
|
||||
for element in alert_config_value:
|
||||
try:
|
||||
formatted_list.append(element.format(**context))
|
||||
except (AttributeError, KeyError, IndexError):
|
||||
formatted_list.append(element)
|
||||
alert_config[alert_config_field] = formatted_list
|
||||
|
||||
return alert_config
|
||||
|
||||
def send_to_thehive(self, alert_config):
|
||||
connection_details = self.rule['hive_connection']
|
||||
api = TheHiveApi(
|
||||
connection_details.get('hive_host', ''),
|
||||
connection_details.get('hive_apikey', ''),
|
||||
proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}),
|
||||
cert=connection_details.get('hive_verify', False))
|
||||
|
||||
alert = Alert(**alert_config)
|
||||
response = api.create_alert(alert)
|
||||
|
||||
if response.status_code != 201:
|
||||
raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
|
||||
|
||||
def alert(self, matches):
|
||||
if self.rule.get('hive_alert_config_type', 'custom') != 'classic':
|
||||
for match in matches:
|
||||
alert_config = self.create_alert_config(match)
|
||||
self.send_to_thehive(alert_config)
|
||||
else:
|
||||
alert_config = self.create_alert_config(matches[0])
|
||||
artifacts = []
|
||||
for match in matches:
|
||||
artifacts += self.create_artifacts(match)
|
||||
if 'related_events' in match:
|
||||
for related_event in match['related_events']:
|
||||
artifacts += self.create_artifacts(related_event)
|
||||
|
||||
alert_config['artifacts'] = artifacts
|
||||
alert_config['title'] = self.create_title(matches)
|
||||
alert_config['description'] = self.create_alert_body(matches)
|
||||
self.send_to_thehive(alert_config)
|
||||
|
||||
def get_info(self):
|
||||
|
||||
return {
|
||||
'type': 'hivealerter',
|
||||
'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '')
|
||||
}
|
||||
@@ -26,7 +26,8 @@ filter:
|
||||
alert: modules.so.thehive.TheHiveAlerter
|
||||
|
||||
hive_connection:
|
||||
hive_host: https://{{hivehost}}/thehive/
|
||||
hive_host: http://{{hivehost}}
|
||||
hive_port: 9000
|
||||
hive_apikey: {{hivekey}}
|
||||
|
||||
hive_proxies:
|
||||
|
||||
@@ -55,35 +55,35 @@ elastalogdir:
|
||||
file.directory:
|
||||
- name: /opt/so/log/elastalert
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- makedirs: True
|
||||
|
||||
elastarules:
|
||||
file.directory:
|
||||
- name: /opt/so/rules/elastalert
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- makedirs: True
|
||||
|
||||
elastaconfdir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/elastalert
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- makedirs: True
|
||||
|
||||
elastasomodulesdir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/elastalert/modules/so
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- makedirs: True
|
||||
|
||||
elastacustmodulesdir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/elastalert/modules/custom
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- makedirs: True
|
||||
|
||||
elastasomodulesync:
|
||||
@@ -91,7 +91,7 @@ elastasomodulesync:
|
||||
- name: /opt/so/conf/elastalert/modules/so
|
||||
- source: salt://elastalert/files/modules/so
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- makedirs: True
|
||||
|
||||
elastarulesync:
|
||||
@@ -99,7 +99,7 @@ elastarulesync:
|
||||
- name: /opt/so/rules/elastalert
|
||||
- source: salt://elastalert/files/rules/so
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- template: jinja
|
||||
|
||||
elastaconf:
|
||||
@@ -107,7 +107,7 @@ elastaconf:
|
||||
- name: /opt/so/conf/elastalert/elastalert_config.yaml
|
||||
- source: salt://elastalert/files/elastalert_config.yaml
|
||||
- user: 933
|
||||
- group: 939
|
||||
- group: 933
|
||||
- template: jinja
|
||||
|
||||
so-elastalert:
|
||||
@@ -118,16 +118,9 @@ so-elastalert:
|
||||
- user: elastalert
|
||||
- detach: True
|
||||
- binds:
|
||||
- /opt/so/rules/elastalert:/etc/elastalert/rules/:ro
|
||||
- /opt/so/rules/elastalert:/opt/rules/:ro
|
||||
- /opt/so/log/elastalert:/var/log/elastalert:rw
|
||||
- /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
|
||||
- /opt/so/conf/elastalert/elastalert_config.yaml:/etc/elastalert/conf/elastalert_config.yaml:ro
|
||||
- environment:
|
||||
- ELASTICSEARCH_HOST: {{ esip }}
|
||||
- ELASTICSEARCH_PORT: {{ esport }}
|
||||
- ELASTALERT_CONFIG: /etc/elastalert/conf/elastalert_config.yaml
|
||||
- ELASTALERT_SUPERVISOR_CONF: /etc/elastalert/conf/elastalert_supervisord.conf
|
||||
- RULES_DIRECTORY: /etc/elastalert/rules/
|
||||
- LOG_DIR: /var/log/elastalert
|
||||
- /opt/so/conf/elastalert/elastalert_config.yaml:/opt/config/elastalert_config.yaml:ro
|
||||
|
||||
{% endif %}
|
||||
|
||||
@@ -1226,7 +1226,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 5"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1365,7 +1365,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 8"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1504,7 +1504,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 8"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1643,7 +1643,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 8"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
|
||||
@@ -290,7 +290,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 16"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -430,7 +430,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 16"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1046,7 +1046,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 16"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1186,7 +1186,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 16"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1326,7 +1326,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 16"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
|
||||
4915
salt/grafana/dashboards/mastersearch/mastersearch.json
Normal file
4915
salt/grafana/dashboards/mastersearch/mastersearch.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -298,7 +298,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 16"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -438,7 +438,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 16"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
|
||||
@@ -1326,7 +1326,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 8"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1465,7 +1465,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 8"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
@@ -1604,7 +1604,7 @@
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
" / 8"
|
||||
" / {{ CPUS }}"
|
||||
],
|
||||
"type": "math"
|
||||
}
|
||||
|
||||
@@ -10,6 +10,13 @@ providers:
|
||||
editable: true
|
||||
options:
|
||||
path: /etc/grafana/grafana_dashboards/master
|
||||
- name: 'Master Search'
|
||||
folder: 'Master Search'
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /etc/grafana/grafana_dashboards/mastersearch
|
||||
- name: 'Sensor Nodes'
|
||||
folder: 'Sensor Nodes'
|
||||
type: file
|
||||
|
||||
@@ -33,6 +33,13 @@ grafanadashmdir:
|
||||
- group: 939
|
||||
- makedirs: True
|
||||
|
||||
grafanadashmsdir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/grafana/grafana_dashboards/mastersearch
|
||||
- user: 939
|
||||
- group: 939
|
||||
- makedirs: True
|
||||
|
||||
grafanadashevaldir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/grafana/grafana_dashboards/eval
|
||||
@@ -85,6 +92,29 @@ dashboard-master:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if salt['pillar.get']('mastersearchtab', False) %}
|
||||
{% for SN, SNDATA in salt['pillar.get']('mastersearchtab', {}).items() %}
|
||||
{% set NODETYPE = SN.split('_')|last %}
|
||||
{% set SN = SN | regex_replace('_' ~ NODETYPE, '') %}
|
||||
dashboard-master:
|
||||
file.managed:
|
||||
- name: /opt/so/conf/grafana/grafana_dashboards/mastersearch/{{ SN }}-MasterSearch.json
|
||||
- user: 939
|
||||
- group: 939
|
||||
- template: jinja
|
||||
- source: salt://grafana/dashboards/mastersearch/mastersearch.json
|
||||
- defaults:
|
||||
SERVERNAME: {{ SN }}
|
||||
MANINT: {{ SNDATA.manint }}
|
||||
MONINT: {{ SNDATA.manint }}
|
||||
CPUS: {{ SNDATA.totalcpus }}
|
||||
UID: {{ SNDATA.guid }}
|
||||
ROOTFS: {{ SNDATA.rootfs }}
|
||||
NSMFS: {{ SNDATA.nsmfs }}
|
||||
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if salt['pillar.get']('sensorstab', False) %}
|
||||
{% for SN, SNDATA in salt['pillar.get']('sensorstab', {}).items() %}
|
||||
{% set NODETYPE = SN.split('_')|last %}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -616,15 +616,14 @@
|
||||
|
||||
|
||||
# # Read stats from one or more Elasticsearch servers or clusters
|
||||
{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %}
|
||||
{% if grains['role'] in ['so-master', 'so-eval', 'so-mastersearch'] %}
|
||||
[[inputs.elasticsearch]]
|
||||
|
||||
# ## specify a list of one or more Elasticsearch servers
|
||||
# # you can add username and password to your url to use basic authentication:
|
||||
# # servers = ["http://user:pass@localhost:9200"]
|
||||
servers = ["http://{{ MASTER }}:9200"]
|
||||
{% endif %}
|
||||
{% if grains['role'] == 'so-node' %}
|
||||
{% elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %}
|
||||
[[inputs.elasticsearch]]
|
||||
servers = ["http://{{ NODEIP }}:9200"]
|
||||
{% endif %}
|
||||
@@ -667,14 +666,14 @@
|
||||
# # Read metrics from one or more commands that can output to stdout
|
||||
|
||||
# ## Commands array
|
||||
{% if grains['role'] == 'so-master' %}
|
||||
{% if grains['role'] in ['so-master', 'so-mastersearch'] %}
|
||||
[[inputs.exec]]
|
||||
commands = [
|
||||
"/scripts/redis.sh",
|
||||
"/scripts/influxdbsize.sh"
|
||||
]
|
||||
data_format = "influx"
|
||||
{% elif grains['role'] == 'so-sensor' %}
|
||||
{% elif grains['role'] in ['so-sensor', 'so-heavynode'] %}
|
||||
[[inputs.exec]]
|
||||
commands = [
|
||||
"/scripts/stenoloss.sh",
|
||||
|
||||
@@ -38,3 +38,9 @@ echo "Applying cross cluster search config..."
|
||||
curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}'
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if salt['pillar.get']('mastersearchtab', {}) %}
|
||||
{%- for SN, SNDATA in salt['pillar.get']('mastersearchtab', {}).items() %}
|
||||
curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}'
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
|
||||
@@ -1325,7 +1325,7 @@ set_initial_firewall_policy() {
|
||||
/opt/so/saltstack/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" bond0
|
||||
;;
|
||||
'MASTERSEARCH')
|
||||
/opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
|
||||
/opt/so/saltstack/pillar/data/addtotab.sh mastersearchtab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
|
||||
Reference in New Issue
Block a user