Merge branch 'dev' of github.com:Security-Onion-Solutions/securityonion-saltstack into dev

pillar/data/mastersearchtab.sls

@@ -0,0 +1 @@
+mastersearchtab:

salt/elastalert/files/modules/so/thehive.py

@@ -1,107 +0,0 @@
-# -*- coding: utf-8 -*-
-# HiveAlerter modified from original at: https://raw.githubusercontent.com/Nclose-ZA/elastalert_hive_alerter/master/elastalert_hive_alerter/hive_alerter.py
-
-import uuid
-
-from elastalert.alerts import Alerter
-from thehive4py.api import TheHiveApi
-from thehive4py.models import Alert, AlertArtifact, CustomFieldHelper
-
-
-class TheHiveAlerter(Alerter):
-    """
-    Use matched data to create alerts containing observables in an instance of TheHive
-    """
-
-    required_options = set(['hive_connection', 'hive_alert_config'])
-
-    def get_aggregation_summary_text(self, matches):
-        text = super(TheHiveAlerter, self).get_aggregation_summary_text(matches)
-        if text:
-            text = '```\n{0}```\n'.format(text)
-        return text
-
-    def create_artifacts(self, match):
-        artifacts = []
-        context = {'rule': self.rule, 'match': match}
-        for mapping in self.rule.get('hive_observable_data_mapping', []):
-            for observable_type, match_data_key in mapping.items():
-                try:
-                    artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context)))
-                except KeyError as e:
-                    print(('format string {} fail cause no key {} in {}'.format(e, match_data_key, context)))
-        return artifacts
-
-    def create_alert_config(self, match):
-        context = {'rule': self.rule, 'match': match}
-        alert_config = {
-            'artifacts': self.create_artifacts(match),
-            'sourceRef': str(uuid.uuid4())[0:6],
-            'title': '{rule[name]}'.format(**context)
-        }
-
-        alert_config.update(self.rule.get('hive_alert_config', {}))
-
-        for alert_config_field, alert_config_value in alert_config.items():
-            if alert_config_field == 'customFields':
-                custom_fields = CustomFieldHelper()
-                for cf_key, cf_value in alert_config_value.items():
-                    try:
-                        func = getattr(custom_fields, 'add_{}'.format(cf_value['type']))
-                    except AttributeError:
-                        raise Exception('unsupported custom field type {}'.format(cf_value['type']))
-                    value = cf_value['value'].format(**context)
-                    func(cf_key, value)
-                alert_config[alert_config_field] = custom_fields.build()
-            elif isinstance(alert_config_value, str):
-                alert_config[alert_config_field] = alert_config_value.format(**context)
-            elif isinstance(alert_config_value, (list, tuple)):
-                formatted_list = []
-                for element in alert_config_value:
-                    try:
-                        formatted_list.append(element.format(**context))
-                    except (AttributeError, KeyError, IndexError):
-                        formatted_list.append(element)
-                alert_config[alert_config_field] = formatted_list
-
-        return alert_config
-
-    def send_to_thehive(self, alert_config):
-        connection_details = self.rule['hive_connection']
-        api = TheHiveApi(
-            connection_details.get('hive_host', ''),
-            connection_details.get('hive_apikey', ''),
-            proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}),
-            cert=connection_details.get('hive_verify', False))
-
-        alert = Alert(**alert_config)
-        response = api.create_alert(alert)
-
-        if response.status_code != 201:
-            raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
-
-    def alert(self, matches):
-        if self.rule.get('hive_alert_config_type', 'custom') != 'classic':
-            for match in matches:
-                alert_config = self.create_alert_config(match)
-                self.send_to_thehive(alert_config)
-        else:
-            alert_config = self.create_alert_config(matches[0])
-            artifacts = []
-            for match in matches:
-                artifacts += self.create_artifacts(match)
-                if 'related_events' in match:
-                    for related_event in match['related_events']:
-                        artifacts += self.create_artifacts(related_event)
-
-            alert_config['artifacts'] = artifacts
-            alert_config['title'] = self.create_title(matches)
-            alert_config['description'] = self.create_alert_body(matches)
-            self.send_to_thehive(alert_config)
-
-    def get_info(self):
-
-        return {
-            'type': 'hivealerter',
-            'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '')
-        }

salt/elastalert/files/rules/so/nids2hive.yaml

@@ -26,7 +26,8 @@ filter:
 alert: modules.so.thehive.TheHiveAlerter
 
 hive_connection:
-  hive_host: https://{{hivehost}}/thehive/
+  hive_host: http://{{hivehost}}
+  hive_port: 9000
   hive_apikey: {{hivekey}}
 
 hive_proxies:

salt/elastalert/init.sls

@@ -55,35 +55,35 @@ elastalogdir:
   file.directory:
     - name: /opt/so/log/elastalert
     - user: 933
-    - group: 939
+    - group: 933
     - makedirs: True
 
 elastarules:
   file.directory:
     - name: /opt/so/rules/elastalert
     - user: 933
-    - group: 939
+    - group: 933
     - makedirs: True
 
 elastaconfdir:
   file.directory:
     - name: /opt/so/conf/elastalert
     - user: 933
-    - group: 939
+    - group: 933
     - makedirs: True
 
 elastasomodulesdir:
   file.directory:
     - name: /opt/so/conf/elastalert/modules/so
     - user: 933
-    - group: 939
+    - group: 933
     - makedirs: True
 
 elastacustmodulesdir:
   file.directory:
     - name: /opt/so/conf/elastalert/modules/custom
     - user: 933
-    - group: 939
+    - group: 933
     - makedirs: True
 
 elastasomodulesync:
@@ -91,7 +91,7 @@ elastasomodulesync:
     - name: /opt/so/conf/elastalert/modules/so
     - source: salt://elastalert/files/modules/so
     - user: 933
-    - group: 939
+    - group: 933
     - makedirs: True
 
 elastarulesync:
@@ -99,7 +99,7 @@ elastarulesync:
     - name: /opt/so/rules/elastalert
     - source: salt://elastalert/files/rules/so
     - user: 933
-    - group: 939
+    - group: 933
     - template: jinja
 
 elastaconf:
@@ -107,7 +107,7 @@ elastaconf:
     - name: /opt/so/conf/elastalert/elastalert_config.yaml
     - source: salt://elastalert/files/elastalert_config.yaml
     - user: 933
-    - group: 939
+    - group: 933
     - template: jinja
 
 so-elastalert:
@@ -118,16 +118,9 @@ so-elastalert:
     - user: elastalert
     - detach: True
     - binds:
-      - /opt/so/rules/elastalert:/etc/elastalert/rules/:ro
+      - /opt/so/rules/elastalert:/opt/rules/:ro
       - /opt/so/log/elastalert:/var/log/elastalert:rw
       - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
-      - /opt/so/conf/elastalert/elastalert_config.yaml:/etc/elastalert/conf/elastalert_config.yaml:ro
+      - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/config/elastalert_config.yaml:ro
-    - environment:
-      - ELASTICSEARCH_HOST: {{ esip }}
-      - ELASTICSEARCH_PORT: {{ esport }}
-      - ELASTALERT_CONFIG: /etc/elastalert/conf/elastalert_config.yaml
-      - ELASTALERT_SUPERVISOR_CONF: /etc/elastalert/conf/elastalert_supervisord.conf
-      - RULES_DIRECTORY: /etc/elastalert/rules/
-      - LOG_DIR: /var/log/elastalert
 
 {% endif %}

salt/grafana/dashboards/eval/eval.json

@@ -1226,7 +1226,7 @@
                 },
                 {
                   "params": [
-                    " / 5"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }
@@ -1365,7 +1365,7 @@
                 },
                 {
                   "params": [
-                    " / 8"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }
@@ -1504,7 +1504,7 @@
                 },
                 {
                   "params": [
-                    " / 8"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }
@@ -1643,7 +1643,7 @@
                 },
                 {
                   "params": [
-                    " / 8"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }

salt/grafana/dashboards/master/master.json

@@ -290,7 +290,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -430,7 +430,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1046,7 +1046,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1186,7 +1186,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1326,7 +1326,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }

salt/grafana/dashboards/search_nodes/searchnode.json

@@ -298,7 +298,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -438,7 +438,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }

salt/grafana/dashboards/sensor_nodes/sensor.json

@@ -1326,7 +1326,7 @@
               },
               {
                 "params": [
-                  " / 8"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1465,7 +1465,7 @@
               },
               {
                 "params": [
-                  " / 8"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1604,7 +1604,7 @@
               },
               {
                 "params": [
-                  " / 8"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }

salt/grafana/etc/dashboards/dashboard.yml

@@ -10,6 +10,13 @@ providers:
   editable: true
   options:
     path: /etc/grafana/grafana_dashboards/master
+- name: 'Master Search'
+  folder: 'Master Search'
+  type: file
+  disableDeletion: false
+  editable: true
+  options:
+    path: /etc/grafana/grafana_dashboards/mastersearch
 - name: 'Sensor Nodes'
   folder: 'Sensor Nodes'
   type: file

salt/grafana/init.sls

@@ -33,6 +33,13 @@ grafanadashmdir:
     - group: 939
     - makedirs: True
 
+grafanadashmsdir:
+  file.directory:
+    - name: /opt/so/conf/grafana/grafana_dashboards/mastersearch
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 grafanadashevaldir:
   file.directory:
     - name: /opt/so/conf/grafana/grafana_dashboards/eval
@@ -85,6 +92,29 @@ dashboard-master:
 {% endfor %}
 {% endif %}
 
+{% if salt['pillar.get']('mastersearchtab', False) %}
+{% for SN, SNDATA in salt['pillar.get']('mastersearchtab', {}).items() %}
+{% set NODETYPE = SN.split('_')|last %}
+{% set SN = SN | regex_replace('_' ~ NODETYPE, '') %}
+dashboard-master:
+  file.managed:
+    - name: /opt/so/conf/grafana/grafana_dashboards/mastersearch/{{ SN }}-MasterSearch.json
+    - user: 939
+    - group: 939
+    - template: jinja
+    - source: salt://grafana/dashboards/mastersearch/mastersearch.json
+    - defaults:
+      SERVERNAME: {{ SN }}
+      MANINT: {{ SNDATA.manint }}
+      MONINT: {{ SNDATA.manint }}
+      CPUS: {{ SNDATA.totalcpus }}
+      UID: {{ SNDATA.guid }}
+      ROOTFS: {{ SNDATA.rootfs }}
+      NSMFS: {{ SNDATA.nsmfs }}
+
+{% endfor %}
+{% endif %}
+
 {% if salt['pillar.get']('sensorstab', False) %}
 {% for SN, SNDATA in salt['pillar.get']('sensorstab', {}).items() %}
 {% set NODETYPE = SN.split('_')|last %}

salt/telegraf/etc/telegraf.conf

@@ -616,15 +616,14 @@
 
 
 # # Read stats from one or more Elasticsearch servers or clusters
-{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %}
+{% if grains['role'] in ['so-master', 'so-eval', 'so-mastersearch']   %}
  [[inputs.elasticsearch]]
 
 #   ## specify a list of one or more Elasticsearch servers
 #   # you can add username and password to your url to use basic authentication:
 #   # servers = ["http://user:pass@localhost:9200"]
    servers = ["http://{{ MASTER }}:9200"]
-{% endif %}
+{% elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode', 'so-heavynode']  %}
-{% if grains['role'] == 'so-node' %}
  [[inputs.elasticsearch]]
    servers = ["http://{{ NODEIP }}:9200"]
 {% endif %}
@@ -667,14 +666,14 @@
 # # Read metrics from one or more commands that can output to stdout
 
 #   ## Commands array
-{% if grains['role'] == 'so-master' %}
+{% if grains['role'] in ['so-master', 'so-mastersearch'] %}
 [[inputs.exec]]
    commands = [
      "/scripts/redis.sh",
      "/scripts/influxdbsize.sh"
    ]
    data_format = "influx"
-{% elif grains['role'] == 'so-sensor' %}
+{% elif grains['role'] in ['so-sensor', 'so-heavynode'] %}
 [[inputs.exec]]
    commands = [
      "/scripts/stenoloss.sh",

salt/utility/bin/crossthestreams

@@ -38,3 +38,9 @@ echo "Applying cross cluster search config..."
 curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}'
   {%- endfor %}
 {%- endif %}
+
+{%- if salt['pillar.get']('mastersearchtab', {}) %}
+  {%- for SN, SNDATA in salt['pillar.get']('mastersearchtab', {}).items() %}
+curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}'
+  {%- endfor %}
+{%- endif %}

setup/so-functions

@@ -1325,7 +1325,7 @@ set_initial_firewall_policy() {
 					/opt/so/saltstack/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" bond0
 					;;
 				'MASTERSEARCH')
-					/opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
+					/opt/so/saltstack/pillar/data/addtotab.sh mastersearchtab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
 					;;
 			esac
 			;;