Fix all scripts for ssl elastic

salt/common/tools/sbin/so-elastic-clear

@@ -50,11 +50,7 @@ done
 if [ $SKIP -ne 1 ]; then
         # List indices
         echo 
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        {% else %}
-        curl -L {{ NODEIP }}:9200/_cat/indices?v
-        {% endif %}
         echo
         # Inform user we are about to delete all data
         echo
@@ -93,18 +89,10 @@ fi
 # Delete data
 echo "Deleting data..."
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
 for INDX in ${INDXS}
 do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% else %}
-    curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% endif %}
 done
 
 #Start Logstash/Filebeat

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
-for p in $ESPORT $THEHIVEESPORT; do
+curl -s -k -XPUT -H "Content-Type: application/json" -L http://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-   curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-done

salt/common/tools/sbin/so-elasticsearch-pipeline-stats

@@ -19,15 +19,7 @@
 . /usr/sbin/so-common
 
 if [ "$1" == "" ]; then
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% endif %}
 else
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-pipelines-list

@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-templates-list

@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
-    {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-templates-load

@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-    {% else %}
-	curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
 
 
 echo "Loading templates..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% else %}
-for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% endif %}
 echo
 
 cd - >/dev/null

salt/common/tools/sbin/so-index-list

@@ -15,8 +15,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 curl -X GET -k -L https://localhost:9200/_cat/indices?v
-{% else %}
-curl -X GET -L localhost:9200/_cat/indices?v
-{% endif %}

salt/curator/files/curator.yml

@@ -12,11 +12,11 @@ client:
     - {{elasticsearch}}
   port: 9200
   url_prefix:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  use_ssl: True{% else %}  use_ssl: False{% endif %}
+  use_ssl: True
   certificate:
   client_cert:
   client_key:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  ssl_no_validate: True{% else %}  ssl_no_validate: False{% endif %}
+  ssl_no_validate: True
   http_auth:
   timeout: 30
   master_only: False

salt/elasticsearch/files/so-elasticsearch-pipelines

@@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
+    curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% else %}
-	curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -51,11 +47,7 @@ fi
 cd ${ELASTICSEARCH_INGEST_PIPELINES}
 
 echo "Loading pipelines..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% else %}
-for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% endif %}
 echo
 
 cd - >/dev/null

salt/filebeat/etc/filebeat.yml

@@ -260,7 +260,7 @@ output.{{ type }}:
   {%- if grains['role'] in ["so-eval", "so-import"] %}
 output.elasticsearch:
   enabled: true
-  hosts: ["{{ MANAGER }}:9200"]
+  hosts: ["https://{{ MANAGER }}:9200"]
   pipelines:
     - pipeline: "%{[module]}.%{[dataset]}"
   indices:

salt/utility/bin/crossthestreams

@@ -1,7 +1,6 @@
 #!/bin/bash
 {% set ES = salt['pillar.get']('manager:mainip', '') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 
 # Wait for ElasticSearch to come up, so that we can query for version infromation
@@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
+  curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
@@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 
 echo "Applying cross cluster search config..."
-    curl -s -XPUT -L https://{{ ES }}:9200/_cluster/settings \
+    curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
          -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
 
@@ -37,7 +36,7 @@ echo "Applying cross cluster search config..."
 {%- if TRUECLUSTER is sameas false %}
   {%- if salt['pillar.get']('nodestab', {}) %}
     {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-curl -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
+curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
     {%- endfor %}
   {%- endif %}
 {%- endif %}

salt/utility/bin/eval

@@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
+  curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
@@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 
 echo "Applying cross cluster search config..."
-    curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+    curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
          -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"