diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear
index 78e80d014..4c7271272 100755
--- a/salt/common/tools/sbin/so-elastic-clear
+++ b/salt/common/tools/sbin/so-elastic-clear
@@ -50,11 +50,7 @@ done
if [ $SKIP -ne 1 ]; then
# List indices
echo
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
- {% else %}
- curl -L {{ NODEIP }}:9200/_cat/indices?v
- {% endif %}
echo
# Inform user we are about to delete all data
echo
@@ -93,18 +89,10 @@ fi
# Delete data
echo "Deleting data..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
for INDX in ${INDXS}
do
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
- {% else %}
- curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
- {% endif %}
done
#Start Logstash/Filebeat
diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw
index 837b22090..6b123bd0d 100755
--- a/salt/common/tools/sbin/so-elasticsearch-indices-rw
+++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw
@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
echo "Removing read only attributes for indices..."
echo
-for p in $ESPORT $THEHIVEESPORT; do
- curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
-done
+curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
index 234be2017..146196917 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
+++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
@@ -19,15 +19,7 @@
. /usr/sbin/so-common
if [ "$1" == "" ]; then
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
- {% endif %}
else
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
- {% endif %}
fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
index 37da36127..565f90071 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list
+++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
@@ -17,15 +17,7 @@
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
. /usr/sbin/so-common
if [ "$1" == "" ]; then
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
- {% endif %}
else
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
- {% endif %}
fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list
index aa011b573..494ca5770 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-list
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-list
@@ -17,15 +17,7 @@
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
. /usr/sbin/so-common
if [ "$1" == "" ]; then
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
- {% endif %}
else
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
- {% endif %}
fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load
index 76558e17a..42a836854 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-load
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-load
@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% else %}
- curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% endif %}
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
echo "Loading templates..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% else %}
-for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% endif %}
echo
cd - >/dev/null
diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable
deleted file mode 100755
index 015b47eba..000000000
--- a/salt/common/tools/sbin/so-features-enable
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-. /usr/sbin/so-common
-. /usr/sbin/so-image-common
-local_salt_dir=/opt/so/saltstack/local
-
-cat << EOF
-This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license.
-If you proceed, then we will download new Docker images and restart services.
-
-Please review the Elastic license:
-https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
-
-Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext!
-(We expect to support Elastic Features Security at some point in the future.)
-
-Do you agree to the terms of the Elastic license and understand the note about encryption?
-
-If so, type AGREE to accept the Elastic license and continue. Otherwise, just press Enter to exit this program without making any changes.
-EOF
-
-read INPUT
-if [ "$INPUT" != "AGREE" ]; then
- exit
-fi
-
-echo "Please wait while switching to Elastic Features."
-
-require_manager
-
-TRUSTED_CONTAINERS=( \
- "so-elasticsearch" \
- "so-filebeat" \
- "so-kibana" \
- "so-logstash" )
-update_docker_containers "features" "-features"
-
-# Modify global.sls to enable Features
-sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls
diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list
index 1ed27a095..dcfebbf58 100755
--- a/salt/common/tools/sbin/so-index-list
+++ b/salt/common/tools/sbin/so-index-list
@@ -15,8 +15,4 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -X GET -k -L https://localhost:9200/_cat/indices?v
-{% else %}
-curl -X GET -L localhost:9200/_cat/indices?v
-{% endif %}
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 6eb01a694..49de2a77a 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -613,16 +613,6 @@ if [ $is_airgap -eq 0 ]; then
else
update_registry
update_docker_containers "soup"
- FEATURESCHECK=$(lookup_pillar features elastic)
- if [[ "$FEATURESCHECK" == "True" ]]; then
- TRUSTED_CONTAINERS=(
- "so-elasticsearch"
- "so-filebeat"
- "so-kibana"
- "so-logstash"
- )
- update_docker_containers "features" "-features"
- fi
fi
echo ""
echo "Stopping Salt Minion service."
diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml
index 016a123f0..7d86ccc04 100644
--- a/salt/curator/files/curator.yml
+++ b/salt/curator/files/curator.yml
@@ -12,11 +12,11 @@ client:
- {{elasticsearch}}
port: 9200
url_prefix:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %} use_ssl: True{% else %} use_ssl: False{% endif %}
+ use_ssl: True
certificate:
client_cert:
client_key:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %} ssl_no_validate: True{% else %} ssl_no_validate: False{% endif %}
+ ssl_no_validate: True
http_auth:
timeout: 30
master_only: False
diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml
index a22d65b7b..ad675b8ee 100644
--- a/salt/elastalert/defaults.yaml
+++ b/salt/elastalert/defaults.yaml
@@ -16,8 +16,8 @@ elastalert:
#aws_region: us-east-1
#profile: test
#es_url_prefix: elasticsearch
- #use_ssl: True
- #verify_certs: True
+ use_ssl: true
+ verify_certs: false
#es_send_get_body_as: GET
#es_username: someusername
#es_password: somepassword
diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls
index 9008832f1..8fcb46cda 100644
--- a/salt/elastalert/init.sls
+++ b/salt/elastalert/init.sls
@@ -104,8 +104,9 @@ elastaconf:
wait_for_elasticsearch:
module.run:
- http.wait_for_successful_query:
- - url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*'
+ - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*'
- wait_for: 180
+ - verify_ssl: False
so-elastalert:
docker_container.running:
diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index 2e20a878b..fed45bf79 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -9,12 +9,6 @@
{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %}
cluster.name: "{{ ESCLUSTERNAME }}"
network.host: 0.0.0.0
-
-# minimum_master_nodes need to be explicitly set when bound on a public IP
-# set to 1 to allow single node clusters
-# Details: https://github.com/elastic/elasticsearch/pull/17288
-#discovery.zen.minimum_master_nodes: 1
-# This is a test -- if this is here, then the volume is mounted correctly.
path.logs: /var/log/elasticsearch
action.destructive_requires_name: true
transport.bind_host: 0.0.0.0
@@ -25,17 +19,12 @@ cluster.routing.allocation.disk.watermark.low: 95%
cluster.routing.allocation.disk.watermark.high: 98%
cluster.routing.allocation.disk.watermark.flood_stage: 98%
xpack.ml.enabled: false
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
xpack.security.enabled: true
-{%- else %}
-xpack.security.enabled: false
-{%- endif %}
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: none
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.client_authentication: none
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
@@ -46,7 +35,6 @@ xpack.security.authc:
username: anonymous_user
roles: superuser
authz_exception: true
-{%- endif %}
node.name: {{ grains.host }}
script.max_compilations_rate: 1000/1m
{%- if TRUECLUSTER is sameas true %}
diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines
index dce6a081b..fca50b7d4 100755
--- a/salt/elasticsearch/files/so-elasticsearch-pipelines
+++ b/salt/elasticsearch/files/so-elasticsearch-pipelines
@@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
- {% if grains['role'] in ['so-node','so-heavynode'] %}
- curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% else %}
- curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% endif %}
+ curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -51,11 +47,7 @@ fi
cd ${ELASTICSEARCH_INGEST_PIPELINES}
echo "Loading pipelines..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% else %}
-for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% endif %}
echo
cd - >/dev/null
diff --git a/salt/elasticsearch/files/sotls.yml b/salt/elasticsearch/files/sotls.yml
deleted file mode 100644
index 2fc577337..000000000
--- a/salt/elasticsearch/files/sotls.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-keystore.path: /usr/share/elasticsearch/config/sokeys
-keystore.password: changeit
-keystore.algorithm: SunX509
-truststore.path: /etc/pki/java/cacerts
-truststore.password: changeit
-truststore.algorithm: PKIX
-protocols:
-- TLSv1.2
-ciphers:
-- TLS_RSA_WITH_AES_128_CBC_SHA256
-- TLS_RSA_WITH_AES_256_GCM_SHA384
-transport.encrypted: true
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
-http.encrypted: true
-{%- else %}
-http.encrypted: false
-{%- endif %}
diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls
index debb37512..df297986a 100644
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -140,14 +140,6 @@ esyml:
- group: 939
- template: jinja
-sotls:
- file.managed:
- - name: /opt/so/conf/elasticsearch/sotls.yml
- - source: salt://elasticsearch/files/sotls.yml
- - user: 930
- - group: 939
- - template: jinja
-
#sync templates to /opt/so/conf/elasticsearch/templates
{% for TEMPLATE in TEMPLATES %}
es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
@@ -199,7 +191,7 @@ so-elasticsearch:
{% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
- discovery.type=single-node
{% endif %}
- - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }}
+ - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true
ulimits:
- memlock=-1:-1
- nofile=65536:65536
@@ -221,7 +213,6 @@ so-elasticsearch:
- /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
- /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
- - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro
- watch:
- file: cacertz
- file: esyml
diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 1b902d542..b6aa218ef 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -260,7 +260,7 @@ output.{{ type }}:
{%- if grains['role'] in ["so-eval", "so-import"] %}
output.elasticsearch:
enabled: true
- hosts: ["{{ MANAGER }}:9200"]
+ hosts: ["https://{{ MANAGER }}:9200"]
pipelines:
- pipeline: "%{[module]}.%{[dataset]}"
indices:
diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index 91c6cdbb8..339d307ee 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -13,7 +13,6 @@
# along with this program. If not, see .
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls in allowed_states %}
-
{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{% set LOCALHOSTNAME = salt['grains.get']('host') %}
diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh
deleted file mode 100644
index 541a666bd..000000000
--- a/salt/kibana/bin/keepkibanahappy.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-# Wait for ElasticSearch to come up, so that we can query for version infromation
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 30 ]]; do
- curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
- if [ $? -eq 0 ]; then
- ELASTICSEARCH_CONNECTED="yes"
- echo "connected!"
- break
- else
- ((COUNT+=1))
- sleep 1
- echo -n "."
- fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
- echo
- echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'"
- echo
-
- exit
-fi
-
-# Make sure Kibana is running
-MAX_WAIT=240
-
-# Check to see if Kibana is available
-wait_step=0
- until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do
- wait_step=$(( ${wait_step} + 1 ))
- echo "Waiting on Kibana...Attempt #$wait_step"
- if [ ${wait_step} -gt ${MAX_WAIT} ]; then
- echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
- exit 5
- fi
- sleep 1s;
- done
-
-
-# Apply Kibana template
- echo
- echo "Applying Kibana template..."
- curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \
- -H 'Content-Type: application/json' \
- -d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}'
- echo
-
- curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \
- -H 'Content-Type: application/json' \
- -d'{"index" : {"number_of_replicas" : 0}}'
- echo
diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load
index e545dc452..108a4f0f6 100644
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -14,5 +14,14 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o
# SOCtopus and Manager
sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson
+## This hackery will be removed with Elastic Auth ##
+
+# Remove the cold cookie if its there
+rm /tmp/cookie.txt
+
+# Let's snag a cookie from Kibana
+curl -c /tmp/cookie.txt -X GET http://localhost:5601/
+THECOOKIE=$(cat /tmp/cookie.txt | grep sid | awk '{print $7}')
+
# Load saved objects
-curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
+curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml
index 4b9424b4c..856f87909 100644
--- a/salt/kibana/etc/kibana.yml
+++ b/salt/kibana/etc/kibana.yml
@@ -1,11 +1,11 @@
---
# Default Kibana configuration from kibana-docker.
{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
server.name: kibana
server.host: "0"
server.basePath: /kibana
-elasticsearch.hosts: [ "http://{{ ES }}:9200" ]
+elasticsearch.hosts: [ "https://{{ ES }}:9200" ]
+elasticsearch.ssl.verificationMode: none
#kibana.index: ".kibana"
#elasticsearch.username: elastic
#elasticsearch.password: changeme
@@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000
logging.dest: /var/log/kibana/kibana.log
telemetry.enabled: false
security.showInsecureClusterWarning: false
+xpack.security.authc.providers:
+ anonymous.anonymous1:
+ order: 0
+ credentials: "elasticsearch_anonymous_user"
diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
index d1764eecc..d17dc2b22 100644
--- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "zeek" and "import" not in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-zeek"
template => "/templates/so-zeek-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
index 35900471e..4562dcee7 100644
--- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if "import" in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-import"
template => "/templates/so-import-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
index 0a7d961de..fb6eaee5d 100644
--- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [event_type] == "sflow" {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-flow"
template => "/templates/so-flow-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
index 7f0e30fbc..61aa21a82 100644
--- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [event_type] == "ids" and "import" not in [tags] {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-ids"
template => "/templates/so-ids-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
index 4a27428f7..0afbf45ea 100644
--- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "syslog" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-syslog"
template => "/templates/so-syslog-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
index 2a71e3fab..3144884d9 100644
--- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "osquery" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-osquery"
template => "/templates/so-osquery-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
index 9b93d327b..764f597b9 100644
--- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [dataset] =~ "firewall" {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-firewall"
template => "/templates/so-firewall-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
index 29837040a..5013bafc1 100644
--- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "suricata" and "import" not in [tags] {
elasticsearch {
@@ -12,10 +11,8 @@ output {
index => "so-ids"
template_name => "so-ids"
template => "/templates/so-ids-template.json"
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
index beaf24727..349c0ada1 100644
--- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if "beat-ext" in [tags] and "import" not in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-beats"
template => "/templates/so-beats-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
index 95c81577b..1a4987a53 100644
--- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "ossec" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-ossec"
template => "/templates/so-ossec-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
index 1e8c44cc6..d564486e4 100644
--- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "strelka" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-strelka"
template => "/templates/so-strelka-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index 0742eb33e..520f87b93 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -1,7 +1,6 @@
{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
{%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %}
{%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
{%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
{%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %}
{%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %}
@@ -31,7 +30,7 @@
"hostUrl": "http://{{ MANAGERIP }}:4434/"
},
"elastic": {
- "hostUrl": "http://{{ MANAGERIP }}:9200",
+ "hostUrl": "https://{{ MANAGERIP }}:9200",
{%- if salt['pillar.get']('nodestab', {}) %}
"remoteHostUrls": [
{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf
index 4f58ecf83..c9941c3e1 100644
--- a/salt/soctopus/files/SOCtopus.conf
+++ b/salt/soctopus/files/SOCtopus.conf
@@ -6,7 +6,7 @@
[es]
-es_url = http://{{MANAGER}}:9200
+es_url = https://{{MANAGER}}:9200
es_ip = {{MANAGER}}
es_user = YOURESUSER
es_pass = YOURESPASS
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index 0c447172f..b8976b8c9 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -622,7 +622,7 @@
# ## specify a list of one or more Elasticsearch servers
# # you can add username and password to your url to use basic authentication:
# # servers = ["http://user:pass@localhost:9200"]
- servers = ["http://{{ MANAGER }}:9200"]
+ servers = ["https://{{ MANAGER }}:9200"]
{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]
diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams
index 490c7b548..3838f67df 100644
--- a/salt/utility/bin/crossthestreams
+++ b/salt/utility/bin/crossthestreams
@@ -1,7 +1,6 @@
#!/bin/bash
{% set ES = salt['pillar.get']('manager:mainip', '') %}
{% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
# Wait for ElasticSearch to come up, so that we can query for version infromation
@@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
- curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
+ curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
fi
echo "Applying cross cluster search config..."
- curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+ curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
-H 'Content-Type: application/json' \
-d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
@@ -37,7 +36,7 @@ echo "Applying cross cluster search config..."
{%- if TRUECLUSTER is sameas false %}
{%- if salt['pillar.get']('nodestab', {}) %}
{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-curl -XPUT -L http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
+curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
{%- endfor %}
{%- endif %}
{%- endif %}
diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval
index f63a61942..dcf46de7a 100644
--- a/salt/utility/bin/eval
+++ b/salt/utility/bin/eval
@@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
- curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
+ curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
fi
echo "Applying cross cluster search config..."
- curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+ curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
-H 'Content-Type: application/json' \
-d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"