Merge pull request #665 from Security-Onion-Solutions/issue/227

Issue/227

pillar/data/mastersearchtab.sls

@@ -0,0 +1 @@
+mastersearchtab:

salt/grafana/dashboards/eval/eval.json

@@ -1226,7 +1226,7 @@
                 },
                 {
                   "params": [
-                    " / 5"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }
@@ -1365,7 +1365,7 @@
                 },
                 {
                   "params": [
-                    " / 8"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }
@@ -1504,7 +1504,7 @@
                 },
                 {
                   "params": [
-                    " / 8"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }
@@ -1643,7 +1643,7 @@
                 },
                 {
                   "params": [
-                    " / 8"
+                    " / {{ CPUS }}"
                   ],
                   "type": "math"
                 }

salt/grafana/dashboards/master/master.json

@@ -290,7 +290,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -430,7 +430,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1046,7 +1046,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1186,7 +1186,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1326,7 +1326,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }

salt/grafana/dashboards/search_nodes/searchnode.json

@@ -298,7 +298,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -438,7 +438,7 @@
               },
               {
                 "params": [
-                  " / 16"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }

salt/grafana/dashboards/sensor_nodes/sensor.json

@@ -1326,7 +1326,7 @@
               },
               {
                 "params": [
-                  " / 8"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1465,7 +1465,7 @@
               },
               {
                 "params": [
-                  " / 8"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }
@@ -1604,7 +1604,7 @@
               },
               {
                 "params": [
-                  " / 8"
+                  " / {{ CPUS }}"
                 ],
                 "type": "math"
               }

salt/grafana/etc/dashboards/dashboard.yml

@@ -10,6 +10,13 @@ providers:
   editable: true
   options:
     path: /etc/grafana/grafana_dashboards/master
+- name: 'Master Search'
+  folder: 'Master Search'
+  type: file
+  disableDeletion: false
+  editable: true
+  options:
+    path: /etc/grafana/grafana_dashboards/mastersearch
 - name: 'Sensor Nodes'
   folder: 'Sensor Nodes'
   type: file

salt/grafana/init.sls

@@ -33,6 +33,13 @@ grafanadashmdir:
     - group: 939
     - makedirs: True
 
+grafanadashmsdir:
+  file.directory:
+    - name: /opt/so/conf/grafana/grafana_dashboards/mastersearch
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 grafanadashevaldir:
   file.directory:
     - name: /opt/so/conf/grafana/grafana_dashboards/eval
@@ -85,6 +92,29 @@ dashboard-master:
 {% endfor %}
 {% endif %}
 
+{% if salt['pillar.get']('mastersearchtab', False) %}
+{% for SN, SNDATA in salt['pillar.get']('mastersearchtab', {}).items() %}
+{% set NODETYPE = SN.split('_')|last %}
+{% set SN = SN | regex_replace('_' ~ NODETYPE, '') %}
+dashboard-master:
+  file.managed:
+    - name: /opt/so/conf/grafana/grafana_dashboards/mastersearch/{{ SN }}-MasterSearch.json
+    - user: 939
+    - group: 939
+    - template: jinja
+    - source: salt://grafana/dashboards/mastersearch/mastersearch.json
+    - defaults:
+      SERVERNAME: {{ SN }}
+      MANINT: {{ SNDATA.manint }}
+      MONINT: {{ SNDATA.manint }}
+      CPUS: {{ SNDATA.totalcpus }}
+      UID: {{ SNDATA.guid }}
+      ROOTFS: {{ SNDATA.rootfs }}
+      NSMFS: {{ SNDATA.nsmfs }}
+
+{% endfor %}
+{% endif %}
+
 {% if salt['pillar.get']('sensorstab', False) %}
 {% for SN, SNDATA in salt['pillar.get']('sensorstab', {}).items() %}
 {% set NODETYPE = SN.split('_')|last %}

salt/telegraf/etc/telegraf.conf

@@ -616,15 +616,14 @@
 
 
 # # Read stats from one or more Elasticsearch servers or clusters
-{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %}
+{% if grains['role'] in ['so-master', 'so-eval', 'so-mastersearch']   %}
  [[inputs.elasticsearch]]
 
 #   ## specify a list of one or more Elasticsearch servers
 #   # you can add username and password to your url to use basic authentication:
 #   # servers = ["http://user:pass@localhost:9200"]
    servers = ["http://{{ MASTER }}:9200"]
-{% endif %}
+{% elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode', 'so-heavynode']  %}
-{% if grains['role'] == 'so-node' %}
  [[inputs.elasticsearch]]
    servers = ["http://{{ NODEIP }}:9200"]
 {% endif %}
@@ -667,14 +666,14 @@
 # # Read metrics from one or more commands that can output to stdout
 
 #   ## Commands array
-{% if grains['role'] == 'so-master' %}
+{% if grains['role'] in ['so-master', 'so-mastersearch'] %}
 [[inputs.exec]]
    commands = [
      "/scripts/redis.sh",
      "/scripts/influxdbsize.sh"
    ]
    data_format = "influx"
-{% elif grains['role'] == 'so-sensor' %}
+{% elif grains['role'] in ['so-sensor', 'so-heavynode'] %}
 [[inputs.exec]]
    commands = [
      "/scripts/stenoloss.sh",

salt/utility/bin/crossthestreams

@@ -38,3 +38,9 @@ echo "Applying cross cluster search config..."
 curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}'
   {%- endfor %}
 {%- endif %}
+
+{%- if salt['pillar.get']('mastersearchtab', {}) %}
+  {%- for SN, SNDATA in salt['pillar.get']('mastersearchtab', {}).items() %}
+curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}'
+  {%- endfor %}
+{%- endif %}

setup/so-functions

@@ -1325,7 +1325,7 @@ set_initial_firewall_policy() {
 					/opt/so/saltstack/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" bond0
 					;;
 				'MASTERSEARCH')
-					/opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
+					/opt/so/saltstack/pillar/data/addtotab.sh mastersearchtab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
 					;;
 			esac
 			;;