diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index 6f3ba495b..1aa445c97 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -3,6 +3,7 @@ logstash: pipelines: manager: config: + - so/0008_input_fleet_livequery.conf.jinja - so/0009_input_beats.conf - so/0010_input_hhbeats.conf - so/9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 7a5aeec39..8ae84fe1f 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -8,6 +8,7 @@ logstash: - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja - so/9100_output_osquery.conf.jinja + - so/9101_output_osquery_livequery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index 78e80d014..4c7271272 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -50,11 +50,7 @@ done if [ $SKIP -ne 1 ]; then # List indices echo - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v - {% else %} - curl -L {{ NODEIP }}:9200/_cat/indices?v - {% endif %} echo # Inform user we are about to delete all data echo @@ -93,18 +89,10 @@ fi # Delete data echo "Deleting data..." -{% if grains['role'] in ['so-node','so-heavynode'] %} INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') -{% else %} -INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') -{% endif %} for INDX in ${INDXS} do - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 - {% else %} - curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 - {% endif %} done #Start Logstash/Filebeat diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw index 837b22090..6b123bd0d 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-rw +++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw @@ -21,6 +21,5 @@ THEHIVEESPORT=9400 echo "Removing read only attributes for indices..." echo -for p in $ESPORT $THEHIVEESPORT; do - curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; -done +curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats index 234be2017..146196917 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats @@ -19,15 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" - {% else %} - curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" - {% else %} - curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list index 37da36127..565f90071 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list @@ -17,15 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' - {% else %} - curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq - {% else %} - curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list index aa011b573..494ca5770 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-templates-list @@ -17,15 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' - {% else %} - curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys' - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq - {% else %} - curl -s -L {{ NODEIP }}:9200/_template/$1 | jq - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 76558e17a..42a836854 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% else %} - curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% endif %} if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES} echo "Loading templates..." -{% if grains['role'] in ['so-node','so-heavynode'] %} for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done -{% else %} -for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done -{% endif %} echo cd - >/dev/null diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable deleted file mode 100755 index 015b47eba..000000000 --- a/salt/common/tools/sbin/so-features-enable +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -. /usr/sbin/so-common -. /usr/sbin/so-image-common -local_salt_dir=/opt/so/saltstack/local - -cat << EOF -This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license. -If you proceed, then we will download new Docker images and restart services. - -Please review the Elastic license: -https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt - -Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext! -(We expect to support Elastic Features Security at some point in the future.) - -Do you agree to the terms of the Elastic license and understand the note about encryption? - -If so, type AGREE to accept the Elastic license and continue. Otherwise, just press Enter to exit this program without making any changes. -EOF - -read INPUT -if [ "$INPUT" != "AGREE" ]; then - exit -fi - -echo "Please wait while switching to Elastic Features." - -require_manager - -TRUSTED_CONTAINERS=( \ - "so-elasticsearch" \ - "so-filebeat" \ - "so-kibana" \ - "so-logstash" ) -update_docker_containers "features" "-features" - -# Modify global.sls to enable Features -sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list index 1ed27a095..dcfebbf58 100755 --- a/salt/common/tools/sbin/so-index-list +++ b/salt/common/tools/sbin/so-index-list @@ -15,8 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% if grains['role'] in ['so-node','so-heavynode'] %} curl -X GET -k -L https://localhost:9200/_cat/indices?v -{% else %} -curl -X GET -L localhost:9200/_cat/indices?v -{% endif %} diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e33d0cc41..64ff5ca6b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -612,16 +612,6 @@ if [ $is_airgap -eq 0 ]; then else update_registry update_docker_containers "soup" - FEATURESCHECK=$(lookup_pillar features elastic) - if [[ "$FEATURESCHECK" == "True" ]]; then - TRUSTED_CONTAINERS=( - "so-elasticsearch" - "so-filebeat" - "so-kibana" - "so-logstash" - ) - update_docker_containers "features" "-features" - fi fi echo "" echo "Stopping Salt Minion service." diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 016a123f0..7d86ccc04 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -12,11 +12,11 @@ client: - {{elasticsearch}} port: 9200 url_prefix: -{% if grains['role'] in ['so-node', 'so-heavynode'] %} use_ssl: True{% else %} use_ssl: False{% endif %} + use_ssl: True certificate: client_cert: client_key: -{% if grains['role'] in ['so-node', 'so-heavynode'] %} ssl_no_validate: True{% else %} ssl_no_validate: False{% endif %} + ssl_no_validate: True http_auth: timeout: 30 master_only: False diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index a22d65b7b..ad675b8ee 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -16,8 +16,8 @@ elastalert: #aws_region: us-east-1 #profile: test #es_url_prefix: elasticsearch - #use_ssl: True - #verify_certs: True + use_ssl: true + verify_certs: false #es_send_get_body_as: GET #es_username: someusername #es_password: somepassword diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 9008832f1..8fcb46cda 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -104,8 +104,9 @@ elastaconf: wait_for_elasticsearch: module.run: - http.wait_for_successful_query: - - url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*' + - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*' - wait_for: 180 + - verify_ssl: False so-elastalert: docker_container.running: diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 2e20a878b..fed45bf79 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -9,12 +9,6 @@ {%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %} cluster.name: "{{ ESCLUSTERNAME }}" network.host: 0.0.0.0 - -# minimum_master_nodes need to be explicitly set when bound on a public IP -# set to 1 to allow single node clusters -# Details: https://github.com/elastic/elasticsearch/pull/17288 -#discovery.zen.minimum_master_nodes: 1 -# This is a test -- if this is here, then the volume is mounted correctly. path.logs: /var/log/elasticsearch action.destructive_requires_name: true transport.bind_host: 0.0.0.0 @@ -25,17 +19,12 @@ cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% xpack.ml.enabled: false -{%- if grains['role'] in ['so-node','so-heavynode'] %} xpack.security.enabled: true -{%- else %} -xpack.security.enabled: false -{%- endif %} xpack.security.transport.ssl.enabled: true xpack.security.transport.ssl.verification_mode: none xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] -{%- if grains['role'] in ['so-node','so-heavynode'] %} xpack.security.http.ssl.enabled: true xpack.security.http.ssl.client_authentication: none xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key @@ -46,7 +35,6 @@ xpack.security.authc: username: anonymous_user roles: superuser authz_exception: true -{%- endif %} node.name: {{ grains.host }} script.max_compilations_rate: 1000/1m {%- if TRUECLUSTER is sameas true %} diff --git a/salt/elasticsearch/files/ingest/osquery.live_query b/salt/elasticsearch/files/ingest/osquery.live_query new file mode 100644 index 000000000..92f325e1d --- /dev/null +++ b/salt/elasticsearch/files/ingest/osquery.live_query @@ -0,0 +1,16 @@ +{ + "description" : "osquery live query", + "processors" : [ + { + "script": { + "lang": "painless", + "source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; " + } + }, + { "remove": { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } }, + { "rename": { "field": "distributed_query_execution_id", "target_field": "result.query_id", "ignore_missing": true } }, + { "rename": { "field": "computer_name", "target_field": "host.hostname", "ignore_missing": true } }, + { "pipeline": { "name": "osquery.normalize" } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/osquery.normalize b/salt/elasticsearch/files/ingest/osquery.normalize new file mode 100644 index 000000000..ce0a6ca92 --- /dev/null +++ b/salt/elasticsearch/files/ingest/osquery.normalize @@ -0,0 +1,14 @@ +{ + "description" : "osquery normalize", + "processors" : [ + { "rename": { "field": "result.columns.cmdline", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "result.columns.cwd", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "result.columns.name", "target_field": "process.name", "ignore_missing": true } }, + { "rename": { "field": "result.columns.path", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "result.columns.pid", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "result.columns.parent", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "result.columns.uid", "target_field": "user.id", "ignore_missing": true } }, + { "rename": { "field": "result.columns.username", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "result.columns.gid", "target_field": "group.id", "ignore_missing": true } } + ] +} diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result index b6b4f22ef..9bb381946 100644 --- a/salt/elasticsearch/files/ingest/osquery.query_result +++ b/salt/elasticsearch/files/ingest/osquery.query_result @@ -1,24 +1,19 @@ { "description" : "osquery", "processors" : [ - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, - { "gsub": { "field": "message2.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } }, - { "rename": { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns", "target_field": "winlog", "ignore_missing": true } }, + { "json": { "field": "message", "target_field": "result", "ignore_failure": true } }, + { "gsub": { "field": "result.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } }, + { "rename": { "if": "ctx.result.columns?.eventid != null", "field": "result.columns", "target_field": "winlog", "ignore_missing": true } }, { "json": { "field": "winlog.data", "target_field": "unparsed", "ignore_failure": true} }, { "set": { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true } }, { "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } }, { "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } }, { "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, - { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { - "script": { - "lang": "painless", - "source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; " - } - }, + { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } }, { "set": { "field": "event.module", "value": "osquery", "override": false } }, - { "set": { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false} }, + { "set": { "field": "event.dataset", "value": "{{result.name}}", "override": false} }, + { "pipeline": { "if": "!(ctx.containsKey('winlog'))", "name": "osquery.normalize" } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index dce6a081b..fca50b7d4 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - {% if grains['role'] in ['so-node','so-heavynode'] %} - curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% else %} - curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% endif %} + curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -51,11 +47,7 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." -{% if grains['role'] in ['so-node','so-heavynode'] %} for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done -{% else %} -for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done -{% endif %} echo cd - >/dev/null diff --git a/salt/elasticsearch/files/sotls.yml b/salt/elasticsearch/files/sotls.yml deleted file mode 100644 index 2fc577337..000000000 --- a/salt/elasticsearch/files/sotls.yml +++ /dev/null @@ -1,17 +0,0 @@ -keystore.path: /usr/share/elasticsearch/config/sokeys -keystore.password: changeit -keystore.algorithm: SunX509 -truststore.path: /etc/pki/java/cacerts -truststore.password: changeit -truststore.algorithm: PKIX -protocols: -- TLSv1.2 -ciphers: -- TLS_RSA_WITH_AES_128_CBC_SHA256 -- TLS_RSA_WITH_AES_256_GCM_SHA384 -transport.encrypted: true -{%- if grains['role'] in ['so-node','so-heavynode'] %} -http.encrypted: true -{%- else %} -http.encrypted: false -{%- endif %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index debb37512..df297986a 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -140,14 +140,6 @@ esyml: - group: 939 - template: jinja -sotls: - file.managed: - - name: /opt/so/conf/elasticsearch/sotls.yml - - source: salt://elasticsearch/files/sotls.yml - - user: 930 - - group: 939 - - template: jinja - #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: @@ -199,7 +191,7 @@ so-elasticsearch: {% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %} - discovery.type=single-node {% endif %} - - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} + - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true ulimits: - memlock=-1:-1 - nofile=65536:65536 @@ -221,7 +213,6 @@ so-elasticsearch: - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro - - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro - watch: - file: cacertz - file: esyml diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 062838670..012c590d0 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -365,6 +365,10 @@ "request":{ "type":"object", "dynamic": true + }, + "result":{ + "type":"object", + "dynamic": true }, "rfb":{ "type":"object", diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 1b902d542..b6aa218ef 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -260,7 +260,7 @@ output.{{ type }}: {%- if grains['role'] in ["so-eval", "so-import"] %} output.elasticsearch: enabled: true - hosts: ["{{ MANAGER }}:9200"] + hosts: ["https://{{ MANAGER }}:9200"] pipelines: - pipeline: "%{[module]}.%{[dataset]}" indices: diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 91c6cdbb8..339d307ee 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -13,7 +13,6 @@ # along with this program. If not, see . {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} - {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set LOCALHOSTNAME = salt['grains.get']('host') %} diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh deleted file mode 100644 index 541a666bd..000000000 --- a/salt/kibana/bin/keepkibanahappy.sh +++ /dev/null @@ -1,53 +0,0 @@ -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -# Wait for ElasticSearch to come up, so that we can query for version infromation -echo -n "Waiting for ElasticSearch..." -COUNT=0 -ELASTICSEARCH_CONNECTED="no" -while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi -done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - - exit -fi - -# Make sure Kibana is running -MAX_WAIT=240 - -# Check to see if Kibana is available -wait_step=0 - until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do - wait_step=$(( ${wait_step} + 1 )) - echo "Waiting on Kibana...Attempt #$wait_step" - if [ ${wait_step} -gt ${MAX_WAIT} ]; then - echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds." - exit 5 - fi - sleep 1s; - done - - -# Apply Kibana template - echo - echo "Applying Kibana template..." - curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \ - -H 'Content-Type: application/json' \ - -d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}' - echo - - curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \ - -H 'Content-Type: application/json' \ - -d'{"index" : {"number_of_replicas" : 0}}' - echo diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index e545dc452..b40227031 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -14,5 +14,9 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson +## This hackery will be removed if using Elastic Auth ## + +# Let's snag a cookie from Kibana +THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # Load saved objects -curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1 +curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1 diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 4b9424b4c..856f87909 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,11 +1,11 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} server.name: kibana server.host: "0" server.basePath: /kibana -elasticsearch.hosts: [ "http://{{ ES }}:9200" ] +elasticsearch.hosts: [ "https://{{ ES }}:9200" ] +elasticsearch.ssl.verificationMode: none #kibana.index: ".kibana" #elasticsearch.username: elastic #elasticsearch.password: changeme @@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000 logging.dest: /var/log/kibana/kibana.log telemetry.enabled: false security.showInsecureClusterWarning: false +xpack.security.authc.providers: + anonymous.anonymous1: + order: 0 + credentials: "elasticsearch_anonymous_user" diff --git a/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja new file mode 100644 index 000000000..83aa0c02d --- /dev/null +++ b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja @@ -0,0 +1,19 @@ +{%- set MANAGER = salt['grains.get']('master') %} +{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} +{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} + +input { + redis { + host => '{{ MANAGER }}' + port => 6379 + data_type => 'pattern_channel' + key => 'results_*' + type => 'live_query' + add_field => { + "module" => "osquery" + "dataset" => "live_query" + } + threads => {{ THREADS }} + batch_count => {{ BATCH }} + } +} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index d1764eecc..d17dc2b22 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-zeek" template => "/templates/so-zeek-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 35900471e..4562dcee7 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "import" in [tags] { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-import" template => "/templates/so-import-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 0a7d961de..fb6eaee5d 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "sflow" { elasticsearch { @@ -12,10 +11,8 @@ output { template_name => "so-flow" template => "/templates/so-flow-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 7f0e30fbc..61aa21a82 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { @@ -12,10 +11,8 @@ output { template_name => "so-ids" template => "/templates/so-ids-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 4a27428f7..0afbf45ea 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "syslog" { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-syslog" template => "/templates/so-syslog-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 2a71e3fab..efa46c7af 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,9 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { - if [module] =~ "osquery" { + if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" @@ -13,10 +12,8 @@ output { template_name => "so-osquery" template => "/templates/so-osquery-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja new file mode 100644 index 000000000..51e691176 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -0,0 +1,43 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} + +filter { + if [type] =~ "live_query" { + + mutate { + rename => { + "[host][hostname]" => "computer_name" + } + } + + prune { + blacklist_names => ["host"] + } + + split { + field => "rows" + } + } +} + + +output { + if [type] =~ "live_query" { + elasticsearch { + pipeline => "osquery.live_query" + hosts => "{{ ES }}" + index => "so-osquery" + template_name => "so-osquery" + template => "/templates/so-osquery-template.json" + template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} + } + } +} diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 9b93d327b..764f597b9 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [dataset] =~ "firewall" { elasticsearch { @@ -12,10 +11,8 @@ output { template_name => "so-firewall" template => "/templates/so-firewall-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 29837040a..5013bafc1 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { @@ -12,10 +11,8 @@ output { index => "so-ids" template_name => "so-ids" template => "/templates/so-ids-template.json" - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index beaf24727..349c0ada1 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-beats" template => "/templates/so-beats-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 95c81577b..1a4987a53 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "ossec" { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-ossec" template => "/templates/so-ossec-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 1e8c44cc6..d564486e4 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "strelka" { elasticsearch { @@ -13,10 +12,8 @@ output { template_name => "so-strelka" template => "/templates/so-strelka-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index b8dc5eb21..5f3a359b5 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -42,6 +42,7 @@ { "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.dataset:mysql | groupby mysql.command"}, { "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.dataset:notice | groupby notice.note notice.message"}, { "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.dataset:ntlm | groupby ntlm.server.dns.name"}, + { "name": "Osquery Live Queries", "description": "Osquery Live Query results grouped by computer name", "query": "event.dataset:live_query | groupby host.hostname"}, { "name": "PE", "description": "PE files list", "query": "event.dataset:pe | groupby file.machine file.os file.subsystem"}, { "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.dataset:radius | groupby user.name.keyword"}, { "name": "RDP", "description": "RDP grouped by client name", "query": "event.dataset:rdp | groupby client.name"}, diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index bda2df431..520f87b93 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -1,7 +1,6 @@ {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %} -{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %} {%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %} @@ -31,7 +30,7 @@ "hostUrl": "http://{{ MANAGERIP }}:4434/" }, "elastic": { - "hostUrl": "http://{{ MANAGERIP }}:9200", + "hostUrl": "https://{{ MANAGERIP }}:9200", {%- if salt['pillar.get']('nodestab', {}) %} "remoteHostUrls": [ {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} @@ -45,7 +44,7 @@ }, "sostatus": { "refreshIntervalMs": 30000, - "offlineThresholdMs": 60000 + "offlineThresholdMs": 900000 }, {% if THEHIVEKEY != '' %} "thehive": { diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 4f58ecf83..c9941c3e1 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -6,7 +6,7 @@ [es] -es_url = http://{{MANAGER}}:9200 +es_url = https://{{MANAGER}}:9200 es_ip = {{MANAGER}} es_user = YOURESUSER es_pass = YOURESPASS diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0c447172f..b8976b8c9 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -622,7 +622,7 @@ # ## specify a list of one or more Elasticsearch servers # # you can add username and password to your url to use basic authentication: # # servers = ["http://user:pass@localhost:9200"] - servers = ["http://{{ MANAGER }}:9200"] + servers = ["https://{{ MANAGER }}:9200"] {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 490c7b548..3838f67df 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -1,7 +1,6 @@ #!/bin/bash {% set ES = salt['pillar.get']('manager:mainip', '') %} {% set MANAGER = salt['grains.get']('master') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} # Wait for ElasticSearch to come up, so that we can query for version infromation @@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 + curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \ + curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" @@ -37,7 +36,7 @@ echo "Applying cross cluster search config..." {%- if TRUECLUSTER is sameas false %} {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -curl -XPUT -L http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' +curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' {%- endfor %} {%- endif %} {%- endif %} diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval index f63a61942..dcf46de7a 100644 --- a/salt/utility/bin/eval +++ b/salt/utility/bin/eval @@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 + curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \ + curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"