From 9ee87a92e99ebdb8b777ff62956be05374008d78 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 11 Oct 2018 19:38:25 -0400 Subject: [PATCH] Getting Close --- pillar/data/addtotab.sh | 15 +++++++++++++++ pillar/data/nodestab.sls | 3 +++ pillar/top.sls | 1 + salt/kibana/init.sls | 8 ++++++++ .../logstash/files/dynamic/9999_output_redis.conf | 3 +-- salt/top.sls | 2 ++ .../bin/crossthestreams.sh | 8 ++++++++ salt/utility/init.sls | 10 ++++++++++ so-setup-network.sh | 5 ++++- 9 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 pillar/data/addtotab.sh create mode 100644 pillar/data/nodestab.sls rename salt/{elasticsearch => utility}/bin/crossthestreams.sh (72%) create mode 100644 salt/utility/init.sls diff --git a/pillar/data/addtotab.sh b/pillar/data/addtotab.sh new file mode 100644 index 000000000..db1b35179 --- /dev/null +++ b/pillar/data/addtotab.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# This script adds sensors/nodes/etc to the nodes tab + +TYPE=$1 +NAME=$2 +IPADDRESS=$3 + +if grep -q $3 "/opt/so/saltstack/pillar/data/$1.sls"; then + echo "Storage Node Already in There" +else + echo " $2:" >> /opt/so/saltstack/pillar/data/$1.sls + echo " - $3" >> /opt/so/saltstack/pillar/data/$1.sls + +fi diff --git a/pillar/data/nodestab.sls b/pillar/data/nodestab.sls new file mode 100644 index 000000000..776c52e6e --- /dev/null +++ b/pillar/data/nodestab.sls @@ -0,0 +1,3 @@ +nodestab: + + diff --git a/pillar/top.sls b/pillar/top.sls index 039ed1947..1884e0088 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -10,6 +10,7 @@ base: - masters.{{ grains.host }} - static - firewall.* + - data 'G@role:so-node': - nodes.schedule diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 9095472bb..ac1343db6 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -74,3 +74,11 @@ so-kibana: - /sys/fs/cgroup:/sys/fs/cgroup:ro - port_bindings: - 0.0.0.0:5601:5601 + +# Keep the setting correct +KibanaHappy: + cmd.script: + shell: /bin/bash + runas: socore + source: salt://kibana/bin/keepkibanahappy.sh + template: jinja diff --git a/salt/logstash/files/dynamic/9999_output_redis.conf b/salt/logstash/files/dynamic/9999_output_redis.conf index 46470c350..1769c651c 100644 --- a/salt/logstash/files/dynamic/9999_output_redis.conf +++ b/salt/logstash/files/dynamic/9999_output_redis.conf @@ -1,9 +1,8 @@ {%- if salt['grains.get']('role') == 'so-master' %} -{%- set master = '172.17.0.6' -%} +{%- set master = salt['pillar.get']('master:mainip', '') -%} {%- set nodetype = 'master' %} {%- else %} {%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %} -{%- set master = grains['master'] %} {%- endif %} output { redis { diff --git a/salt/top.sls b/salt/top.sls index 1b6f8c84f..a6a4dd344 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -24,6 +24,8 @@ base: - redis - elasticsearch - logstash + - kibana + - utility # Storage node logic diff --git a/salt/elasticsearch/bin/crossthestreams.sh b/salt/utility/bin/crossthestreams.sh similarity index 72% rename from salt/elasticsearch/bin/crossthestreams.sh rename to salt/utility/bin/crossthestreams.sh index 79fd78936..5997f985e 100644 --- a/salt/elasticsearch/bin/crossthestreams.sh +++ b/salt/utility/bin/crossthestreams.sh @@ -1,4 +1,5 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- set MASTER = grains['master'] %} # Wait for ElasticSearch to come up, so that we can query for version infromation echo -n "Waiting for ElasticSearch..." COUNT=0 @@ -23,5 +24,12 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then exit fi +echo "Applying cross cluster search config..." + curl -s -XPUT http://{{ ES }}:9200/_cluster/settings \ + -H 'Content-Type: application/json' \ + -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MASTER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" + # Add all the storage nodes to cross cluster searching. +{%- for SN, SNIP in salt['pillar.get']('nodestab', {}).iteritems() %}} curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNIP }}:9200"]}}}}}' +{%- endfor %} diff --git a/salt/utility/init.sls b/salt/utility/init.sls new file mode 100644 index 000000000..c2a7799fa --- /dev/null +++ b/salt/utility/init.sls @@ -0,0 +1,10 @@ +# This state is for checking things +{% if grains['role'] == 'so-master' %} + +# Make sure Cross Cluster is good. Will need some logic once we have hot/warm +crossclusterson: + cmd.script: + shell: /bin/bash + runas: socore + source: salt://utility/bin/crossthestreams.sh + template: jinja diff --git a/so-setup-network.sh b/so-setup-network.sh index b7094d3d4..4a8094bb8 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -577,6 +577,7 @@ salt_master_directories() { # Copy over the salt code and templates cp -R pillar/* /opt/so/saltstack/pillar/ chmod +x /opt/so/saltstack/pillar/firewall/addfirewall.sh + chmod +x /opt/so/saltstack/pillar/data/addtotab.sh cp -R salt/* /opt/so/saltstack/salt/ } @@ -639,6 +640,7 @@ set_initial_firewall_policy() { if [ $INSTALLTYPE == 'STORAGENODE' ]; then ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP + ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then @@ -694,7 +696,8 @@ update_sudoers() { # Update Sudoers so that socore can accept keys without a password echo "socore ALL=(ALL) NOPASSWD:/usr/bin/salt-key" | sudo tee -a /etc/sudoers echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/firewall/addfirewall.sh" | sudo tee -a /etc/sudoers - + echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/data/addtotab.sh" | sudo tee -a /etc/sudoers + } ###########################################