diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index 78e80d014..4c7271272 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -50,11 +50,7 @@ done if [ $SKIP -ne 1 ]; then # List indices echo - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v - {% else %} - curl -L {{ NODEIP }}:9200/_cat/indices?v - {% endif %} echo # Inform user we are about to delete all data echo @@ -93,18 +89,10 @@ fi # Delete data echo "Deleting data..." -{% if grains['role'] in ['so-node','so-heavynode'] %} INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') -{% else %} -INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') -{% endif %} for INDX in ${INDXS} do - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 - {% else %} - curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 - {% endif %} done #Start Logstash/Filebeat diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw index 837b22090..6b123bd0d 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-rw +++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw @@ -21,6 +21,5 @@ THEHIVEESPORT=9400 echo "Removing read only attributes for indices..." echo -for p in $ESPORT $THEHIVEESPORT; do - curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; -done +curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats index 234be2017..146196917 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats @@ -19,15 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" - {% else %} - curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" - {% else %} - curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list index 37da36127..565f90071 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list @@ -17,15 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' - {% else %} - curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq - {% else %} - curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list index aa011b573..494ca5770 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-templates-list @@ -17,15 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' - {% else %} - curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys' - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq - {% else %} - curl -s -L {{ NODEIP }}:9200/_template/$1 | jq - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 76558e17a..42a836854 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% else %} - curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% endif %} if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES} echo "Loading templates..." -{% if grains['role'] in ['so-node','so-heavynode'] %} for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done -{% else %} -for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done -{% endif %} echo cd - >/dev/null diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable deleted file mode 100755 index 015b47eba..000000000 --- a/salt/common/tools/sbin/so-features-enable +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -. /usr/sbin/so-common -. /usr/sbin/so-image-common -local_salt_dir=/opt/so/saltstack/local - -cat << EOF -This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license. -If you proceed, then we will download new Docker images and restart services. - -Please review the Elastic license: -https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt - -Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext! -(We expect to support Elastic Features Security at some point in the future.) - -Do you agree to the terms of the Elastic license and understand the note about encryption? - -If so, type AGREE to accept the Elastic license and continue. Otherwise, just press Enter to exit this program without making any changes. -EOF - -read INPUT -if [ "$INPUT" != "AGREE" ]; then - exit -fi - -echo "Please wait while switching to Elastic Features." - -require_manager - -TRUSTED_CONTAINERS=( \ - "so-elasticsearch" \ - "so-filebeat" \ - "so-kibana" \ - "so-logstash" ) -update_docker_containers "features" "-features" - -# Modify global.sls to enable Features -sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list index 1ed27a095..dcfebbf58 100755 --- a/salt/common/tools/sbin/so-index-list +++ b/salt/common/tools/sbin/so-index-list @@ -15,8 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% if grains['role'] in ['so-node','so-heavynode'] %} curl -X GET -k -L https://localhost:9200/_cat/indices?v -{% else %} -curl -X GET -L localhost:9200/_cat/indices?v -{% endif %} diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6eb01a694..49de2a77a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -613,16 +613,6 @@ if [ $is_airgap -eq 0 ]; then else update_registry update_docker_containers "soup" - FEATURESCHECK=$(lookup_pillar features elastic) - if [[ "$FEATURESCHECK" == "True" ]]; then - TRUSTED_CONTAINERS=( - "so-elasticsearch" - "so-filebeat" - "so-kibana" - "so-logstash" - ) - update_docker_containers "features" "-features" - fi fi echo "" echo "Stopping Salt Minion service." diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 016a123f0..7d86ccc04 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -12,11 +12,11 @@ client: - {{elasticsearch}} port: 9200 url_prefix: -{% if grains['role'] in ['so-node', 'so-heavynode'] %} use_ssl: True{% else %} use_ssl: False{% endif %} + use_ssl: True certificate: client_cert: client_key: -{% if grains['role'] in ['so-node', 'so-heavynode'] %} ssl_no_validate: True{% else %} ssl_no_validate: False{% endif %} + ssl_no_validate: True http_auth: timeout: 30 master_only: False diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index a22d65b7b..ad675b8ee 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -16,8 +16,8 @@ elastalert: #aws_region: us-east-1 #profile: test #es_url_prefix: elasticsearch - #use_ssl: True - #verify_certs: True + use_ssl: true + verify_certs: false #es_send_get_body_as: GET #es_username: someusername #es_password: somepassword diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 9008832f1..8fcb46cda 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -104,8 +104,9 @@ elastaconf: wait_for_elasticsearch: module.run: - http.wait_for_successful_query: - - url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*' + - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*' - wait_for: 180 + - verify_ssl: False so-elastalert: docker_container.running: diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 2e20a878b..fed45bf79 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -9,12 +9,6 @@ {%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %} cluster.name: "{{ ESCLUSTERNAME }}" network.host: 0.0.0.0 - -# minimum_master_nodes need to be explicitly set when bound on a public IP -# set to 1 to allow single node clusters -# Details: https://github.com/elastic/elasticsearch/pull/17288 -#discovery.zen.minimum_master_nodes: 1 -# This is a test -- if this is here, then the volume is mounted correctly. path.logs: /var/log/elasticsearch action.destructive_requires_name: true transport.bind_host: 0.0.0.0 @@ -25,17 +19,12 @@ cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% xpack.ml.enabled: false -{%- if grains['role'] in ['so-node','so-heavynode'] %} xpack.security.enabled: true -{%- else %} -xpack.security.enabled: false -{%- endif %} xpack.security.transport.ssl.enabled: true xpack.security.transport.ssl.verification_mode: none xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] -{%- if grains['role'] in ['so-node','so-heavynode'] %} xpack.security.http.ssl.enabled: true xpack.security.http.ssl.client_authentication: none xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key @@ -46,7 +35,6 @@ xpack.security.authc: username: anonymous_user roles: superuser authz_exception: true -{%- endif %} node.name: {{ grains.host }} script.max_compilations_rate: 1000/1m {%- if TRUECLUSTER is sameas true %} diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index dce6a081b..fca50b7d4 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - {% if grains['role'] in ['so-node','so-heavynode'] %} - curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% else %} - curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% endif %} + curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -51,11 +47,7 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." -{% if grains['role'] in ['so-node','so-heavynode'] %} for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done -{% else %} -for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done -{% endif %} echo cd - >/dev/null diff --git a/salt/elasticsearch/files/sotls.yml b/salt/elasticsearch/files/sotls.yml deleted file mode 100644 index 2fc577337..000000000 --- a/salt/elasticsearch/files/sotls.yml +++ /dev/null @@ -1,17 +0,0 @@ -keystore.path: /usr/share/elasticsearch/config/sokeys -keystore.password: changeit -keystore.algorithm: SunX509 -truststore.path: /etc/pki/java/cacerts -truststore.password: changeit -truststore.algorithm: PKIX -protocols: -- TLSv1.2 -ciphers: -- TLS_RSA_WITH_AES_128_CBC_SHA256 -- TLS_RSA_WITH_AES_256_GCM_SHA384 -transport.encrypted: true -{%- if grains['role'] in ['so-node','so-heavynode'] %} -http.encrypted: true -{%- else %} -http.encrypted: false -{%- endif %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index debb37512..df297986a 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -140,14 +140,6 @@ esyml: - group: 939 - template: jinja -sotls: - file.managed: - - name: /opt/so/conf/elasticsearch/sotls.yml - - source: salt://elasticsearch/files/sotls.yml - - user: 930 - - group: 939 - - template: jinja - #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: @@ -199,7 +191,7 @@ so-elasticsearch: {% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %} - discovery.type=single-node {% endif %} - - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} + - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true ulimits: - memlock=-1:-1 - nofile=65536:65536 @@ -221,7 +213,6 @@ so-elasticsearch: - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro - - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro - watch: - file: cacertz - file: esyml diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 1b902d542..b6aa218ef 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -260,7 +260,7 @@ output.{{ type }}: {%- if grains['role'] in ["so-eval", "so-import"] %} output.elasticsearch: enabled: true - hosts: ["{{ MANAGER }}:9200"] + hosts: ["https://{{ MANAGER }}:9200"] pipelines: - pipeline: "%{[module]}.%{[dataset]}" indices: diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 91c6cdbb8..339d307ee 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -13,7 +13,6 @@ # along with this program. If not, see . {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} - {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set LOCALHOSTNAME = salt['grains.get']('host') %} diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh deleted file mode 100644 index 541a666bd..000000000 --- a/salt/kibana/bin/keepkibanahappy.sh +++ /dev/null @@ -1,53 +0,0 @@ -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -# Wait for ElasticSearch to come up, so that we can query for version infromation -echo -n "Waiting for ElasticSearch..." -COUNT=0 -ELASTICSEARCH_CONNECTED="no" -while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi -done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - - exit -fi - -# Make sure Kibana is running -MAX_WAIT=240 - -# Check to see if Kibana is available -wait_step=0 - until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do - wait_step=$(( ${wait_step} + 1 )) - echo "Waiting on Kibana...Attempt #$wait_step" - if [ ${wait_step} -gt ${MAX_WAIT} ]; then - echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds." - exit 5 - fi - sleep 1s; - done - - -# Apply Kibana template - echo - echo "Applying Kibana template..." - curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \ - -H 'Content-Type: application/json' \ - -d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}' - echo - - curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \ - -H 'Content-Type: application/json' \ - -d'{"index" : {"number_of_replicas" : 0}}' - echo diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index e545dc452..108a4f0f6 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -14,5 +14,14 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson +## This hackery will be removed with Elastic Auth ## + +# Remove the cold cookie if its there +rm /tmp/cookie.txt + +# Let's snag a cookie from Kibana +curl -c /tmp/cookie.txt -X GET http://localhost:5601/ +THECOOKIE=$(cat /tmp/cookie.txt | grep sid | awk '{print $7}') + # Load saved objects -curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1 +curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1 diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 4b9424b4c..856f87909 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,11 +1,11 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} server.name: kibana server.host: "0" server.basePath: /kibana -elasticsearch.hosts: [ "http://{{ ES }}:9200" ] +elasticsearch.hosts: [ "https://{{ ES }}:9200" ] +elasticsearch.ssl.verificationMode: none #kibana.index: ".kibana" #elasticsearch.username: elastic #elasticsearch.password: changeme @@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000 logging.dest: /var/log/kibana/kibana.log telemetry.enabled: false security.showInsecureClusterWarning: false +xpack.security.authc.providers: + anonymous.anonymous1: + order: 0 + credentials: "elasticsearch_anonymous_user" diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index d1764eecc..d17dc2b22 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-zeek" template => "/templates/so-zeek-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 35900471e..4562dcee7 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "import" in [tags] { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-import" template => "/templates/so-import-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 0a7d961de..fb6eaee5d 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "sflow" { elasticsearch { @@ -12,10 +11,8 @@ output { template_name => "so-flow" template => "/templates/so-flow-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 7f0e30fbc..61aa21a82 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { @@ -12,10 +11,8 @@ output { template_name => "so-ids" template => "/templates/so-ids-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 4a27428f7..0afbf45ea 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "syslog" { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-syslog" template => "/templates/so-syslog-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 2a71e3fab..3144884d9 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "osquery" { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-osquery" template => "/templates/so-osquery-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 9b93d327b..764f597b9 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [dataset] =~ "firewall" { elasticsearch { @@ -12,10 +11,8 @@ output { template_name => "so-firewall" template => "/templates/so-firewall-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 29837040a..5013bafc1 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { @@ -12,10 +11,8 @@ output { index => "so-ids" template_name => "so-ids" template => "/templates/so-ids-template.json" - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index beaf24727..349c0ada1 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-beats" template => "/templates/so-beats-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 95c81577b..1a4987a53 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "ossec" { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-ossec" template => "/templates/so-ossec-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 1e8c44cc6..d564486e4 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "strelka" { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-strelka" template => "/templates/so-strelka-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 0742eb33e..520f87b93 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -1,7 +1,6 @@ {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %} -{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %} {%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %} @@ -31,7 +30,7 @@ "hostUrl": "http://{{ MANAGERIP }}:4434/" }, "elastic": { - "hostUrl": "http://{{ MANAGERIP }}:9200", + "hostUrl": "https://{{ MANAGERIP }}:9200", {%- if salt['pillar.get']('nodestab', {}) %} "remoteHostUrls": [ {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 4f58ecf83..c9941c3e1 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -6,7 +6,7 @@ [es] -es_url = http://{{MANAGER}}:9200 +es_url = https://{{MANAGER}}:9200 es_ip = {{MANAGER}} es_user = YOURESUSER es_pass = YOURESPASS diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0c447172f..b8976b8c9 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -622,7 +622,7 @@ # ## specify a list of one or more Elasticsearch servers # # you can add username and password to your url to use basic authentication: # # servers = ["http://user:pass@localhost:9200"] - servers = ["http://{{ MANAGER }}:9200"] + servers = ["https://{{ MANAGER }}:9200"] {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 490c7b548..3838f67df 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -1,7 +1,6 @@ #!/bin/bash {% set ES = salt['pillar.get']('manager:mainip', '') %} {% set MANAGER = salt['grains.get']('master') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} # Wait for ElasticSearch to come up, so that we can query for version infromation @@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 + curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \ + curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" @@ -37,7 +36,7 @@ echo "Applying cross cluster search config..." {%- if TRUECLUSTER is sameas false %} {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -curl -XPUT -L http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' +curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' {%- endfor %} {%- endif %} {%- endif %} diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval index f63a61942..dcf46de7a 100644 --- a/salt/utility/bin/eval +++ b/salt/utility/bin/eval @@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 + curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \ + curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"