diff --git a/pillar/top.sls b/pillar/top.sls index a795e03c1..8bd67c174 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -46,6 +46,9 @@ base: - logstash.manager - logstash.search - elasticsearch.search +{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} + - elasticsearch.auth +{% endif %} - data.* - zeeklogs - secrets @@ -89,4 +92,4 @@ base: - secrets - elasticsearch.eval - global - - minions.{{ grains.id }} \ No newline at end of file + - minions.{{ grains.id }} diff --git a/salt/common/init.sls b/salt/common/init.sls index 79a7c5300..d61806a1a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -2,6 +2,7 @@ {% if sls in allowed_states %} {% set role = grains.id.split('_') | last %} +{% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -176,6 +177,9 @@ utilsyncscripts: - file_mode: 755 - template: jinja - source: salt://common/tools/sbin + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} + {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} # Add sensor cleanup diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 56123f418..b48f84b90 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -486,13 +486,14 @@ wait_for_web_response() { url=$1 expected=$2 maxAttempts=${3:-300} + curlcmd=${4:-curl} logfile=/root/wait_for_web_response.log truncate -s 0 "$logfile" attempt=0 while [[ $attempt -lt $maxAttempts ]]; do attempt=$((attempt+1)) echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)" - result=$(curl -ks -L $url) + result=$($curlcmd -ks -L $url) exitcode=$? echo "--------------------------------------------------" >> $logfile diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index 4c7271272..56b5c3d2c 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -50,7 +50,7 @@ done if [ $SKIP -ne 1 ]; then # List indices echo - curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v + {{ ELASTICCURL }} -k -L https://{{ NODEIP }}:9200/_cat/indices?v echo # Inform user we are about to delete all data echo @@ -89,10 +89,10 @@ fi # Delete data echo "Deleting data..." -INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') +INDXS=$({{ ELASTICCURL }} -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') for INDX in ${INDXS} do - curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 + {{ ELASTICCURL }} -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 done #Start Logstash/Filebeat diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-list b/salt/common/tools/sbin/so-elasticsearch-indices-list index c9df67a25..b5cd1b359 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-list +++ b/salt/common/tools/sbin/so-elasticsearch-indices-list @@ -18,4 +18,4 @@ . /usr/sbin/so-common -curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty +{{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw index 6b123bd0d..f5296f2b8 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-rw +++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw @@ -21,5 +21,5 @@ THEHIVEESPORT=9400 echo "Removing read only attributes for indices..." echo -curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; -curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +{{ ELASTICCURL }} -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +{{ ELASTICCURL }} -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats index 146196917..2f9edb6c1 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats @@ -19,7 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" else - curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view index 04901e122..9f799c07f 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -19,7 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . else - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list index 565f90071..f6ef516ef 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list @@ -17,7 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' else - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq fi diff --git a/salt/common/tools/sbin/so-elasticsearch-shards-list b/salt/common/tools/sbin/so-elasticsearch-shards-list index 9d28ed95b..a240f993f 100755 --- a/salt/common/tools/sbin/so-elasticsearch-shards-list +++ b/salt/common/tools/sbin/so-elasticsearch-shards-list @@ -18,4 +18,4 @@ . /usr/sbin/so-common -curl -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty +{{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-template-remove b/salt/common/tools/sbin/so-elasticsearch-template-remove index f7c3e6812..fe19a9d03 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-remove +++ b/salt/common/tools/sbin/so-elasticsearch-template-remove @@ -18,4 +18,4 @@ . /usr/sbin/so-common -curl -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 +{{ ELASTICCURL }} -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 diff --git a/salt/common/tools/sbin/so-elasticsearch-template-view b/salt/common/tools/sbin/so-elasticsearch-template-view index c9f3ec199..1083cb762 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-view +++ b/salt/common/tools/sbin/so-elasticsearch-template-view @@ -19,7 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . else - curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list index 494ca5770..6a7c4d039 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-templates-list @@ -17,7 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' else - curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 42a836854..30ab66b48 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -30,7 +30,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -51,7 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES} echo "Loading templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; {{ ELASTICCURL }} -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done echo cd - >/dev/null diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list index cf9232150..e24599f0e 100755 --- a/salt/common/tools/sbin/so-index-list +++ b/salt/common/tools/sbin/so-index-list @@ -15,4 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -curl -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" +{{ ELASTICCURL }} -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/common/tools/sbin/so-kibana-space-defaults index edf356d45..48225e2f4 100755 --- a/salt/common/tools/sbin/so-kibana-space-defaults +++ b/salt/common/tools/sbin/so-kibana-space-defaults @@ -1,13 +1,13 @@ . /usr/sbin/so-common -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" +wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" ## This hackery will be removed if using Elastic Auth ## # Let's snag a cookie from Kibana -THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') +THECOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Space:" -curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log -echo \ No newline at end of file +{{ ELASTICCURL }} -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log +echo diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 620737c16..14e2148d0 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -326,7 +326,7 @@ rc1_to_rc2() { local NAME=$(echo $p | awk '{print $1}') local IP=$(echo $p | awk '{print $2}') echo "Removing the old cross cluster config for $NAME" - curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}' + {{ ELASTICCURL }} -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}' done /dev/null) + INDICES=$({{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null) [ $? -eq 1 ] && return false echo ${INDICES} | grep -q -E "(logstash-|so-)" } @@ -49,10 +49,10 @@ while overlimit && closedindices; do # First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed. # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. # Finally, select the first entry in that sorted list. - OLDEST_INDEX=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) + OLDEST_INDEX=$({{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. - curl -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} + {{ ELASTICCURL }} -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} # Finally, write a log entry that says we deleted it. echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG} diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 245b700d0..966b0b857 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -5,6 +5,7 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone'] %} + {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # Curator # Create the group curatorgroup: @@ -66,6 +67,8 @@ curcloseddeldel: - group: 939 - mode: 755 - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} curclose: file.managed: @@ -147,4 +150,4 @@ append_so-curator_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/elasticsearch/auth.map.jinja b/salt/elasticsearch/auth.map.jinja new file mode 100644 index 000000000..c6f099025 --- /dev/null +++ b/salt/elasticsearch/auth.map.jinja @@ -0,0 +1,7 @@ +{% set ELASTICAUTH = salt['pillar.filter_by']({ + True: { + 'user': salt['pillar.get']('elasticsearch:auth:user'), + 'pass': salt['pillar.get']('elasticsearch:auth:pass'), + 'elasticcurl':'curl -K /opt/so/conf/elasticsearch/curl.config' }, + False: {'elasticcurl': 'curl'}, +}, pillar='elasticsearch:auth:enabled', default=False) %} diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls new file mode 100644 index 000000000..e8ab1e378 --- /dev/null +++ b/salt/elasticsearch/auth.sls @@ -0,0 +1,12 @@ +elastic_auth_pillar: + file.managed: + - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls + - contents: | + elasticsearch: + auth: + enabled: False + user: so_elastic + pass: {{ salt['random.get_str'](20) }} + # since we are generating a random password, and we don't want that to happen everytime + # a highstate runs, we only manage the file if it doesn't exist + - unless: ls /opt/so/saltstack/local/pillar/elasticsearch/auth.sls diff --git a/salt/elasticsearch/files/curl.config b/salt/elasticsearch/files/curl.config new file mode 100644 index 000000000..e4633df6c --- /dev/null +++ b/salt/elasticsearch/files/curl.config @@ -0,0 +1 @@ +user = "salt['pillar.get']('elasticsearch:auth:user'):salt['pillar.get']('elasticsearch:auth:pass')" diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index fca50b7d4..5d103963e 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -27,7 +27,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -47,9 +47,9 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." -for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done +for i in *; do echo $i; RESPONSE=$({{ ELASTICCURL }} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done echo cd - >/dev/null -exit $RETURN_CODE \ No newline at end of file +exit $RETURN_CODE diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 5ee936f1c..7f287fa27 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -35,6 +35,10 @@ {% endif %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} + +# used in this state to control who can run the so-users script +{% set ES_INCLUDED_NODES = ['so-standalone'] %} vm.max_map_count: sysctl.present: @@ -169,6 +173,7 @@ eslogdir: - group: 939 - makedirs: True +{% if grains.role in ES_INCLUDED_NODES %} # Must run before elasticsearch docker container is started! syncesusers: cmd.run: @@ -176,6 +181,25 @@ syncesusers: - creates: - /opt/so/saltstack/local/salt/elasticsearch/files/users - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles +{% endif %} + +auth_users: + file.managed: + - name: /opt/so/conf/elasticsearch/users + - source: salt://elasticsearch/files/users + - require: +{% if grains.role in ES_INCLUDED_NODES %} + - cmd: syncesusers +{% endif %} + +auth_users_roles: + file.managed: + - name: /opt/so/conf/elasticsearch/users_roles + - source: salt://elasticsearch/files/users_roles +{% if grains.role in ES_INCLUDED_NODES %} + - require: + - cmd: syncesusers +{% endif %} so-elasticsearch: docker_container.running: @@ -223,6 +247,10 @@ so-elasticsearch: - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro + {% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} + - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro + - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro + {% endif %} - watch: - file: cacertz - file: esyml @@ -242,6 +270,8 @@ so-elasticsearch-pipelines-file: - group: 939 - mode: 754 - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} so-elasticsearch-pipelines: cmd.run: @@ -259,6 +289,13 @@ so-elasticsearch-templates: - template: jinja {% endif %} +elastic_curl_config: + file.managed: + - name: /opt/so/conf/elasticsearch/curl.config + - mode: 600 + - contents: user = "{{ salt['pillar.get']('elasticsearch:auth:user') }}:{{ salt['pillar.get']('elasticsearch:auth:pass') }}" + - show_changes: False + {% endif %} {# if grains['role'] != 'so-helix' #} {% else %} diff --git a/salt/firewall/portgroups.yaml b/salt/firewall/portgroups.yaml index 1386267f5..3a7316e98 100644 --- a/salt/firewall/portgroups.yaml +++ b/salt/firewall/portgroups.yaml @@ -89,6 +89,9 @@ firewall: - 514 udp: - 514 + vault: + tcp: + - 8200 wazuh_agent: tcp: - 1514 diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index f07377018..8f80b5c3d 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -16,11 +16,11 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" +#wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" ## This hackery will be removed if using Elastic Auth ## # Let's snag a cookie from Kibana -THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') +THECOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # Load saved objects -curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log \ No newline at end of file +{{ ELASTICCURL }} -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 75b96b72a..61498cfeb 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -4,6 +4,7 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} +{% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # Add ES Group kibanasearchgroup: @@ -63,6 +64,8 @@ kibanabin: - source: salt://kibana/bin/so-kibana-config-load - mode: 755 - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} # Start the kibana docker so-kibana: @@ -113,4 +116,4 @@ so-kibana-config-load: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/top.sls b/salt/top.sls index 8a12aaa26..f3f2958ae 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -238,6 +238,7 @@ base: {%- endif %} {%- if ELASTICSEARCH %} - elasticsearch + - elasticsearch.auth {%- endif %} {%- if LOGSTASH %} - logstash diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 3838f67df..0b2d17918 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -8,7 +8,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -28,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ + {{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" @@ -36,7 +36,7 @@ echo "Applying cross cluster search config..." {%- if TRUECLUSTER is sameas false %} {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' +{{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' {%- endfor %} {%- endif %} {%- endif %} diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval index dcf46de7a..eba0df039 100644 --- a/salt/utility/bin/eval +++ b/salt/utility/bin/eval @@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ + {{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" diff --git a/salt/utility/init.sls b/salt/utility/init.sls index d8b8539fa..0b4f0f2c3 100644 --- a/salt/utility/init.sls +++ b/salt/utility/init.sls @@ -1,8 +1,10 @@ {% from 'allowed_states.map.jinja' import allowed_states %} + {% if sls in allowed_states %} + {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # This state is for checking things -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} + {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} # Make sure Cross Cluster is good. Will need some logic once we have hot/warm crossclusterson: cmd.script: @@ -11,9 +13,11 @@ crossclusterson: - runas: socore - source: salt://utility/bin/crossthestreams - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} -{% endif %} -{% if grains['role'] in ['so-eval', 'so-import'] %} + {% endif %} + {% if grains['role'] in ['so-eval', 'so-import'] %} fixsearch: cmd.script: - shell: /bin/bash @@ -21,7 +25,9 @@ fixsearch: - runas: socore - source: salt://utility/bin/eval - template: jinja -{% endif %} + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} + {% endif %} {% else %}