From 4f867e53756a382e7d09bcfceb247a9901718f1f Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 10:02:23 -0500
Subject: [PATCH] Fix all scripts for ssl elastic

---
 salt/common/tools/sbin/so-elastic-clear              | 12 ------------
 salt/common/tools/sbin/so-elasticsearch-indices-rw   |  5 ++---
 .../tools/sbin/so-elasticsearch-pipeline-stats       |  8 --------
 .../tools/sbin/so-elasticsearch-pipelines-list       |  8 --------
 .../tools/sbin/so-elasticsearch-templates-list       |  8 --------
 .../tools/sbin/so-elasticsearch-templates-load       |  8 --------
 salt/common/tools/sbin/so-index-list                 |  4 ----
 salt/curator/files/curator.yml                       |  4 ++--
 salt/elasticsearch/files/so-elasticsearch-pipelines  | 10 +---------
 salt/filebeat/etc/filebeat.yml                       |  2 +-
 salt/utility/bin/crossthestreams                     |  7 +++----
 salt/utility/bin/eval                                |  4 ++--
 12 files changed, 11 insertions(+), 69 deletions(-)

diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear
index 78e80d014..4c7271272 100755
--- a/salt/common/tools/sbin/so-elastic-clear
+++ b/salt/common/tools/sbin/so-elastic-clear
@@ -50,11 +50,7 @@ done
 if [ $SKIP -ne 1 ]; then
         # List indices
         echo 
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        {% else %}
-        curl -L {{ NODEIP }}:9200/_cat/indices?v
-        {% endif %}
         echo
         # Inform user we are about to delete all data
         echo
@@ -93,18 +89,10 @@ fi
 # Delete data
 echo "Deleting data..."
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
 for INDX in ${INDXS}
 do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% else %}
-    curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% endif %}
 done
 
 #Start Logstash/Filebeat
diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw
index 837b22090..a858db51e 100755
--- a/salt/common/tools/sbin/so-elasticsearch-indices-rw
+++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw
@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
-for p in $ESPORT $THEHIVEESPORT; do
-   curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-done
+curl -s -k -XPUT -H "Content-Type: application/json" -L http://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
index 234be2017..146196917 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
+++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
@@ -19,15 +19,7 @@
 . /usr/sbin/so-common
 
 if [ "$1" == "" ]; then
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% endif %}
 else
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% endif %}
 fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
index 37da36127..565f90071 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list
+++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% endif %}
 fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list
index aa011b573..494ca5770 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-list
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-list
@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
-    {% endif %}
 fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load
index 76558e17a..42a836854 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-load
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-load
@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-    {% else %}
-	curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
 
 
 echo "Loading templates..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% else %}
-for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% endif %}
 echo
 
 cd - >/dev/null
diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list
index 1ed27a095..dcfebbf58 100755
--- a/salt/common/tools/sbin/so-index-list
+++ b/salt/common/tools/sbin/so-index-list
@@ -15,8 +15,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 curl -X GET -k -L https://localhost:9200/_cat/indices?v
-{% else %}
-curl -X GET -L localhost:9200/_cat/indices?v
-{% endif %}
diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml
index 016a123f0..7d86ccc04 100644
--- a/salt/curator/files/curator.yml
+++ b/salt/curator/files/curator.yml
@@ -12,11 +12,11 @@ client:
     - {{elasticsearch}}
   port: 9200
   url_prefix:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  use_ssl: True{% else %}  use_ssl: False{% endif %}
+  use_ssl: True
   certificate:
   client_cert:
   client_key:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  ssl_no_validate: True{% else %}  ssl_no_validate: False{% endif %}
+  ssl_no_validate: True
   http_auth:
   timeout: 30
   master_only: False
diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines
index dce6a081b..fca50b7d4 100755
--- a/salt/elasticsearch/files/so-elasticsearch-pipelines
+++ b/salt/elasticsearch/files/so-elasticsearch-pipelines
@@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
-	curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% else %}
-	curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
+    curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -51,11 +47,7 @@ fi
 cd ${ELASTICSEARCH_INGEST_PIPELINES}
 
 echo "Loading pipelines..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% else %}
-for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% endif %}
 echo
 
 cd - >/dev/null
diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 1b902d542..b6aa218ef 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -260,7 +260,7 @@ output.{{ type }}:
   {%- if grains['role'] in ["so-eval", "so-import"] %}
 output.elasticsearch:
   enabled: true
-  hosts: ["{{ MANAGER }}:9200"]
+  hosts: ["https://{{ MANAGER }}:9200"]
   pipelines:
     - pipeline: "%{[module]}.%{[dataset]}"
   indices:
diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams
index 5fed02fa9..3838f67df 100644
--- a/salt/utility/bin/crossthestreams
+++ b/salt/utility/bin/crossthestreams
@@ -1,7 +1,6 @@
 #!/bin/bash
 {% set ES = salt['pillar.get']('manager:mainip', '') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 
 # Wait for ElasticSearch to come up, so that we can query for version infromation
@@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
+  curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
@@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 
 echo "Applying cross cluster search config..."
-    curl -s -XPUT -L https://{{ ES }}:9200/_cluster/settings \
+    curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
          -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
 
@@ -37,7 +36,7 @@ echo "Applying cross cluster search config..."
 {%- if TRUECLUSTER is sameas false %}
   {%- if salt['pillar.get']('nodestab', {}) %}
     {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-curl -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
+curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
     {%- endfor %}
   {%- endif %}
 {%- endif %}
diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval
index c4af1cfbc..dcf46de7a 100644
--- a/salt/utility/bin/eval
+++ b/salt/utility/bin/eval
@@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
+  curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
@@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 
 echo "Applying cross cluster search config..."
-    curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+    curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
          -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"