From cf21200a36d6dbd383d82545fd96f476bdcd7b9a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 2 Mar 2021 09:11:49 -0500 Subject: [PATCH 01/15] Move version to 2.3.31 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0f1c3e555..dbe139d42 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.40 +2.3.31 From 4014dbbc3ddc5a24c82303c98b4edf196d7b3c82 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 2 Mar 2021 10:14:45 -0500 Subject: [PATCH 02/15] Revert "Move version to 2.3.31" This reverts commit cf21200a36d6dbd383d82545fd96f476bdcd7b9a. --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index dbe139d42..0f1c3e555 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.31 +2.3.40 From 3219f4cd125f2f9235a6ae1fb937b68ab1268fcf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Mar 2021 11:04:50 -0500 Subject: [PATCH 03/15] Remove Features Option --- salt/elasticsearch/files/elasticsearch.yml | 27 ++++++++++------------ salt/elasticsearch/init.sls | 9 +------- salt/filebeat/init.sls | 9 ++------ salt/kibana/init.sls | 8 +------ salt/logstash/init.sls | 9 +------- 5 files changed, 17 insertions(+), 45 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 25bb6cb02..cdd348f2b 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -1,6 +1,5 @@ {%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip') %} -{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} {%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} {%- if TRUECLUSTER is sameas true %} {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name') %} @@ -25,21 +24,19 @@ cluster.routing.allocation.disk.threshold_enabled: true cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% -{%- if FEATURES is sameas true %} xpack.ml.enabled: false -#xpack.security.enabled: false -#xpack.security.transport.ssl.enabled: true -#xpack.security.transport.ssl.verification_mode: none -#xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -#xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -#xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] -#{%- if grains['role'] in ['so-node','so-heavynode'] %} -#xpack.security.http.ssl.enabled: true -#xpack.security.http.ssl.client_authentication: none -#xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -#xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -#xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt -#{%- endif %} +xpack.security.enabled: false +xpack.security.transport.ssl.enabled: true +xpack.security.transport.ssl.verification_mode: none +xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key +xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt +xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] +{%- if grains['role'] in ['so-node','so-heavynode'] %} +xpack.security.http.ssl.enabled: true +xpack.security.http.ssl.client_authentication: none +xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key +xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt +xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt #xpack.security.authc: # anonymous: # username: anonymous_user diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 82fc7c77d..debb37512 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -18,17 +18,10 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} {% set MANAGERIP = salt['pillar.get']('global:managerip') %} -{% if FEATURES is sameas true %} - {% set FEATUREZ = "-features" %} -{% else %} - {% set FEATUREZ = '' %} -{% endif %} - {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %} {% set esclustername = salt['pillar.get']('manager:esclustername') %} {% set esheap = salt['pillar.get']('manager:esheap') %} @@ -186,7 +179,7 @@ eslogdir: so-elasticsearch: docker_container.running: - - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}{{ FEATUREZ }} + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} - hostname: elasticsearch - name: so-elasticsearch - user: elasticsearch diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index f280309f0..e24c572f6 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -21,12 +21,7 @@ {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{%- if FEATURES is sameas true %} - {% set FEATURES = "-features" %} -{% else %} - {% set FEATURES = '' %} -{% endif %} + filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc @@ -64,7 +59,7 @@ filebeatconfsync: OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} so-filebeat: docker_container.running: - - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}{{ FEATURES }} + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} - hostname: so-filebeat - user: root - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }} diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 10b799e80..fe579ffaa 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -4,12 +4,6 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{%- if FEATURES is sameas true %} - {% set FEATURES = "-features" %} -{% else %} - {% set FEATURES = '' %} -{% endif %} # Add ES Group kibanasearchgroup: @@ -73,7 +67,7 @@ kibanabin: # Start the kibana docker so-kibana: docker_container.running: - - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}{{ FEATURES }} + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }} - hostname: kibana - user: kibana - environment: diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 61f533865..2c2c89626 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -19,13 +19,6 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} - -{%- if FEATURES is sameas true %} - {% set FEATURES = "-features" %} -{% else %} - {% set FEATURES = '' %} -{% endif %} # Logstash Section - Decide which pillar to use {% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %} @@ -146,7 +139,7 @@ lslogdir: so-logstash: docker_container.running: - - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}{{ FEATURES }} + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }} - hostname: so-logstash - name: so-logstash - user: logstash From bfd05a8cfc27bd6f43f0951c3d6909c044663e75 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Mar 2021 11:32:29 -0500 Subject: [PATCH 04/15] Change to https for elastic connections --- salt/common/tools/sbin/so-features-enable | 53 ------------------- salt/common/tools/sbin/soup | 10 ---- salt/elastalert/init.sls | 2 +- salt/kibana/bin/keepkibanahappy.sh | 2 +- salt/kibana/etc/kibana.yml | 3 +- .../config/so/9000_output_zeek.conf.jinja | 1 - .../config/so/9002_output_import.conf.jinja | 1 - .../config/so/9004_output_flow.conf.jinja | 1 - .../config/so/9033_output_snort.conf.jinja | 1 - .../config/so/9034_output_syslog.conf.jinja | 1 - .../config/so/9100_output_osquery.conf.jinja | 1 - .../config/so/9200_output_firewall.conf.jinja | 1 - .../config/so/9400_output_suricata.conf.jinja | 1 - .../config/so/9500_output_beats.conf.jinja | 1 - .../config/so/9600_output_ossec.conf.jinja | 1 - .../config/so/9700_output_strelka.conf.jinja | 1 - salt/soc/files/soc/soc.json | 3 +- salt/soctopus/files/SOCtopus.conf | 2 +- salt/telegraf/etc/telegraf.conf | 2 +- salt/utility/bin/crossthestreams | 6 +-- salt/utility/bin/eval | 2 +- 21 files changed, 10 insertions(+), 86 deletions(-) delete mode 100755 salt/common/tools/sbin/so-features-enable diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable deleted file mode 100755 index 015b47eba..000000000 --- a/salt/common/tools/sbin/so-features-enable +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -. /usr/sbin/so-common -. /usr/sbin/so-image-common -local_salt_dir=/opt/so/saltstack/local - -cat << EOF -This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license. -If you proceed, then we will download new Docker images and restart services. - -Please review the Elastic license: -https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt - -Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext! -(We expect to support Elastic Features Security at some point in the future.) - -Do you agree to the terms of the Elastic license and understand the note about encryption? - -If so, type AGREE to accept the Elastic license and continue. Otherwise, just press Enter to exit this program without making any changes. -EOF - -read INPUT -if [ "$INPUT" != "AGREE" ]; then - exit -fi - -echo "Please wait while switching to Elastic Features." - -require_manager - -TRUSTED_CONTAINERS=( \ - "so-elasticsearch" \ - "so-filebeat" \ - "so-kibana" \ - "so-logstash" ) -update_docker_containers "features" "-features" - -# Modify global.sls to enable Features -sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6eb01a694..49de2a77a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -613,16 +613,6 @@ if [ $is_airgap -eq 0 ]; then else update_registry update_docker_containers "soup" - FEATURESCHECK=$(lookup_pillar features elastic) - if [[ "$FEATURESCHECK" == "True" ]]; then - TRUSTED_CONTAINERS=( - "so-elasticsearch" - "so-filebeat" - "so-kibana" - "so-logstash" - ) - update_docker_containers "features" "-features" - fi fi echo "" echo "Stopping Salt Minion service." diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 9008832f1..d51a77128 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -104,7 +104,7 @@ elastaconf: wait_for_elasticsearch: module.run: - http.wait_for_successful_query: - - url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*' + - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*' - wait_for: 180 so-elastalert: diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh index 541a666bd..5d98ca7a0 100644 --- a/salt/kibana/bin/keepkibanahappy.sh +++ b/salt/kibana/bin/keepkibanahappy.sh @@ -4,7 +4,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 + curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 4b9424b4c..ee34e70b1 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,11 +1,10 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} server.name: kibana server.host: "0" server.basePath: /kibana -elasticsearch.hosts: [ "http://{{ ES }}:9200" ] +elasticsearch.hosts: [ "https://{{ ES }}:9200" ] #kibana.index: ".kibana" #elasticsearch.username: elastic #elasticsearch.password: changeme diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index d1764eecc..b8e136a52 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 35900471e..a8acadc89 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "import" in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 0a7d961de..f2e36b072 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "sflow" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 7f0e30fbc..e75ea677a 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 4a27428f7..7470cf096 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "syslog" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 2a71e3fab..b3c2d8e75 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "osquery" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 9b93d327b..e2c75509e 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [dataset] =~ "firewall" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 29837040a..e267cba0a 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index beaf24727..c4c2d57f6 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 95c81577b..4f55a032b 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "ossec" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 1e8c44cc6..a125853ac 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "strelka" { elasticsearch { diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index bda2df431..cf90e5a9d 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -1,7 +1,6 @@ {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %} -{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %} {%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %} @@ -31,7 +30,7 @@ "hostUrl": "http://{{ MANAGERIP }}:4434/" }, "elastic": { - "hostUrl": "http://{{ MANAGERIP }}:9200", + "hostUrl": "https://{{ MANAGERIP }}:9200", {%- if salt['pillar.get']('nodestab', {}) %} "remoteHostUrls": [ {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 4f58ecf83..c9941c3e1 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -6,7 +6,7 @@ [es] -es_url = http://{{MANAGER}}:9200 +es_url = https://{{MANAGER}}:9200 es_ip = {{MANAGER}} es_user = YOURESUSER es_pass = YOURESPASS diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0c447172f..b8976b8c9 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -622,7 +622,7 @@ # ## specify a list of one or more Elasticsearch servers # # you can add username and password to your url to use basic authentication: # # servers = ["http://user:pass@localhost:9200"] - servers = ["http://{{ MANAGER }}:9200"] + servers = ["https://{{ MANAGER }}:9200"] {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 490c7b548..5fed02fa9 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -9,7 +9,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 + curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -29,7 +29,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \ + curl -s -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" @@ -37,7 +37,7 @@ echo "Applying cross cluster search config..." {%- if TRUECLUSTER is sameas false %} {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -curl -XPUT -L http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' +curl -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' {%- endfor %} {%- endif %} {%- endif %} diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval index f63a61942..c4af1cfbc 100644 --- a/salt/utility/bin/eval +++ b/salt/utility/bin/eval @@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200 + curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" From 80574d3c202fe46850af043d1184bb322c692102 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Mar 2021 13:59:43 -0500 Subject: [PATCH 05/15] Make https default for all things --- salt/elasticsearch/files/elasticsearch.yml | 2 -- salt/elasticsearch/files/sotls.yml | 17 ----------------- 2 files changed, 19 deletions(-) delete mode 100644 salt/elasticsearch/files/sotls.yml diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index cdd348f2b..bf9f60c60 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -31,7 +31,6 @@ xpack.security.transport.ssl.verification_mode: none xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] -{%- if grains['role'] in ['so-node','so-heavynode'] %} xpack.security.http.ssl.enabled: true xpack.security.http.ssl.client_authentication: none xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key @@ -42,7 +41,6 @@ xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config # username: anonymous_user # roles: superuser # authz_exception: true -{%- endif %} node.name: {{ grains.host }} script.max_compilations_rate: 1000/1m {%- if TRUECLUSTER is sameas true %} diff --git a/salt/elasticsearch/files/sotls.yml b/salt/elasticsearch/files/sotls.yml deleted file mode 100644 index 2fc577337..000000000 --- a/salt/elasticsearch/files/sotls.yml +++ /dev/null @@ -1,17 +0,0 @@ -keystore.path: /usr/share/elasticsearch/config/sokeys -keystore.password: changeit -keystore.algorithm: SunX509 -truststore.path: /etc/pki/java/cacerts -truststore.password: changeit -truststore.algorithm: PKIX -protocols: -- TLSv1.2 -ciphers: -- TLS_RSA_WITH_AES_128_CBC_SHA256 -- TLS_RSA_WITH_AES_256_GCM_SHA384 -transport.encrypted: true -{%- if grains['role'] in ['so-node','so-heavynode'] %} -http.encrypted: true -{%- else %} -http.encrypted: false -{%- endif %} From e0d9212e55ebbf9814b0a8a05893b8ae875fbe88 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Mar 2021 14:01:05 -0500 Subject: [PATCH 06/15] Make https default for all things --- salt/elasticsearch/init.sls | 9 --------- 1 file changed, 9 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index debb37512..bfed1242a 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -140,14 +140,6 @@ esyml: - group: 939 - template: jinja -sotls: - file.managed: - - name: /opt/so/conf/elasticsearch/sotls.yml - - source: salt://elasticsearch/files/sotls.yml - - user: 930 - - group: 939 - - template: jinja - #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: @@ -221,7 +213,6 @@ so-elasticsearch: - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro - - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro - watch: - file: cacertz - file: esyml From 4f867e53756a382e7d09bcfceb247a9901718f1f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 10:02:23 -0500 Subject: [PATCH 07/15] Fix all scripts for ssl elastic --- salt/common/tools/sbin/so-elastic-clear | 12 ------------ salt/common/tools/sbin/so-elasticsearch-indices-rw | 5 ++--- .../tools/sbin/so-elasticsearch-pipeline-stats | 8 -------- .../tools/sbin/so-elasticsearch-pipelines-list | 8 -------- .../tools/sbin/so-elasticsearch-templates-list | 8 -------- .../tools/sbin/so-elasticsearch-templates-load | 8 -------- salt/common/tools/sbin/so-index-list | 4 ---- salt/curator/files/curator.yml | 4 ++-- salt/elasticsearch/files/so-elasticsearch-pipelines | 10 +--------- salt/filebeat/etc/filebeat.yml | 2 +- salt/utility/bin/crossthestreams | 7 +++---- salt/utility/bin/eval | 4 ++-- 12 files changed, 11 insertions(+), 69 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index 78e80d014..4c7271272 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -50,11 +50,7 @@ done if [ $SKIP -ne 1 ]; then # List indices echo - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v - {% else %} - curl -L {{ NODEIP }}:9200/_cat/indices?v - {% endif %} echo # Inform user we are about to delete all data echo @@ -93,18 +89,10 @@ fi # Delete data echo "Deleting data..." -{% if grains['role'] in ['so-node','so-heavynode'] %} INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') -{% else %} -INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') -{% endif %} for INDX in ${INDXS} do - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 - {% else %} - curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 - {% endif %} done #Start Logstash/Filebeat diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw index 837b22090..a858db51e 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-rw +++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw @@ -21,6 +21,5 @@ THEHIVEESPORT=9400 echo "Removing read only attributes for indices..." echo -for p in $ESPORT $THEHIVEESPORT; do - curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; -done +curl -s -k -XPUT -H "Content-Type: application/json" -L http://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats index 234be2017..146196917 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats @@ -19,15 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" - {% else %} - curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" - {% else %} - curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list index 37da36127..565f90071 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list @@ -17,15 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' - {% else %} - curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq - {% else %} - curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list index aa011b573..494ca5770 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-templates-list @@ -17,15 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' - {% else %} - curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys' - {% endif %} else - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq - {% else %} - curl -s -L {{ NODEIP }}:9200/_template/$1 | jq - {% endif %} fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 76558e17a..42a836854 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - {% if grains['role'] in ['so-node','so-heavynode'] %} curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% else %} - curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% endif %} if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES} echo "Loading templates..." -{% if grains['role'] in ['so-node','so-heavynode'] %} for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done -{% else %} -for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done -{% endif %} echo cd - >/dev/null diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list index 1ed27a095..dcfebbf58 100755 --- a/salt/common/tools/sbin/so-index-list +++ b/salt/common/tools/sbin/so-index-list @@ -15,8 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% if grains['role'] in ['so-node','so-heavynode'] %} curl -X GET -k -L https://localhost:9200/_cat/indices?v -{% else %} -curl -X GET -L localhost:9200/_cat/indices?v -{% endif %} diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 016a123f0..7d86ccc04 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -12,11 +12,11 @@ client: - {{elasticsearch}} port: 9200 url_prefix: -{% if grains['role'] in ['so-node', 'so-heavynode'] %} use_ssl: True{% else %} use_ssl: False{% endif %} + use_ssl: True certificate: client_cert: client_key: -{% if grains['role'] in ['so-node', 'so-heavynode'] %} ssl_no_validate: True{% else %} ssl_no_validate: False{% endif %} + ssl_no_validate: True http_auth: timeout: 30 master_only: False diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index dce6a081b..fca50b7d4 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - {% if grains['role'] in ['so-node','so-heavynode'] %} - curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% else %} - curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - {% endif %} + curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -51,11 +47,7 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." -{% if grains['role'] in ['so-node','so-heavynode'] %} for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done -{% else %} -for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done -{% endif %} echo cd - >/dev/null diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 1b902d542..b6aa218ef 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -260,7 +260,7 @@ output.{{ type }}: {%- if grains['role'] in ["so-eval", "so-import"] %} output.elasticsearch: enabled: true - hosts: ["{{ MANAGER }}:9200"] + hosts: ["https://{{ MANAGER }}:9200"] pipelines: - pipeline: "%{[module]}.%{[dataset]}" indices: diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 5fed02fa9..3838f67df 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -1,7 +1,6 @@ #!/bin/bash {% set ES = salt['pillar.get']('manager:mainip', '') %} {% set MANAGER = salt['grains.get']('master') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} # Wait for ElasticSearch to come up, so that we can query for version infromation @@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 + curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -XPUT -L https://{{ ES }}:9200/_cluster/settings \ + curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" @@ -37,7 +36,7 @@ echo "Applying cross cluster search config..." {%- if TRUECLUSTER is sameas false %} {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -curl -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' +curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' {%- endfor %} {%- endif %} {%- endif %} diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval index c4af1cfbc..dcf46de7a 100644 --- a/salt/utility/bin/eval +++ b/salt/utility/bin/eval @@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 + curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Applying cross cluster search config..." - curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \ + curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" From 07b5cc3d1d24342db04074d01ef8d1425505bb19 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 10:29:41 -0500 Subject: [PATCH 08/15] Fix https for rw indicies script --- salt/common/tools/sbin/so-elasticsearch-indices-rw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw index a858db51e..6b123bd0d 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-rw +++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw @@ -21,5 +21,5 @@ THEHIVEESPORT=9400 echo "Removing read only attributes for indices..." echo -curl -s -k -XPUT -H "Content-Type: application/json" -L http://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; From 4f34eca5b94336ca44d62ee4b1ed6e8e90a1fc31 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 10:32:23 -0500 Subject: [PATCH 09/15] remove unused script --- salt/kibana/bin/keepkibanahappy.sh | 53 ------------------------------ 1 file changed, 53 deletions(-) delete mode 100644 salt/kibana/bin/keepkibanahappy.sh diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh deleted file mode 100644 index 5d98ca7a0..000000000 --- a/salt/kibana/bin/keepkibanahappy.sh +++ /dev/null @@ -1,53 +0,0 @@ -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -# Wait for ElasticSearch to come up, so that we can query for version infromation -echo -n "Waiting for ElasticSearch..." -COUNT=0 -ELASTICSEARCH_CONNECTED="no" -while [[ "$COUNT" -le 30 ]]; do - curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200 - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi -done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - - exit -fi - -# Make sure Kibana is running -MAX_WAIT=240 - -# Check to see if Kibana is available -wait_step=0 - until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do - wait_step=$(( ${wait_step} + 1 )) - echo "Waiting on Kibana...Attempt #$wait_step" - if [ ${wait_step} -gt ${MAX_WAIT} ]; then - echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds." - exit 5 - fi - sleep 1s; - done - - -# Apply Kibana template - echo - echo "Applying Kibana template..." - curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \ - -H 'Content-Type: application/json' \ - -d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}' - echo - - curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \ - -H 'Content-Type: application/json' \ - -d'{"index" : {"number_of_replicas" : 0}}' - echo From 4de62c878ccb26e470a667fb042c54a1fb394594 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 12:51:29 -0500 Subject: [PATCH 10/15] turn on elastic security --- salt/elasticsearch/files/elasticsearch.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index bf9f60c60..a83014a1f 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -25,7 +25,7 @@ cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% xpack.ml.enabled: false -xpack.security.enabled: false +xpack.security.enabled: true xpack.security.transport.ssl.enabled: true xpack.security.transport.ssl.verification_mode: none xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key @@ -36,11 +36,11 @@ xpack.security.http.ssl.client_authentication: none xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt -#xpack.security.authc: -# anonymous: -# username: anonymous_user -# roles: superuser -# authz_exception: true +xpack.security.authc: + anonymous: + username: anonymous_user + roles: superuser + authz_exception: true node.name: {{ grains.host }} script.max_compilations_rate: 1000/1m {%- if TRUECLUSTER is sameas true %} From 6c7111cd0aec19ce57f555e14ac9d24b4ede3d90 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 13:42:04 -0500 Subject: [PATCH 11/15] turn off verification mode for ES --- salt/kibana/etc/kibana.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index ee34e70b1..f0de0084f 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -5,6 +5,7 @@ server.name: kibana server.host: "0" server.basePath: /kibana elasticsearch.hosts: [ "https://{{ ES }}:9200" ] +elasticsearch.ssl.verificationMode: none #kibana.index: ".kibana" #elasticsearch.username: elastic #elasticsearch.password: changeme From e0538417f1cec9f280f367d4907098164ee392bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 14:06:35 -0500 Subject: [PATCH 12/15] fix http.wait --- salt/elastalert/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index d51a77128..8fcb46cda 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -106,6 +106,7 @@ wait_for_elasticsearch: - http.wait_for_successful_query: - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*' - wait_for: 180 + - verify_ssl: False so-elastalert: docker_container.running: From bf4249d28b0f28450a14aec29df7227e92cfd0b7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 14:16:10 -0500 Subject: [PATCH 13/15] fix elastalert verification --- salt/elastalert/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index a22d65b7b..ad675b8ee 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -16,8 +16,8 @@ elastalert: #aws_region: us-east-1 #profile: test #es_url_prefix: elasticsearch - #use_ssl: True - #verify_certs: True + use_ssl: true + verify_certs: false #es_send_get_body_as: GET #es_username: someusername #es_password: somepassword From 49371a1d6a09aa301f2307abdcdd0ce56a574379 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 Mar 2021 14:30:45 -0500 Subject: [PATCH 14/15] fix elastic output for ssl --- salt/kibana/etc/kibana.yml | 4 ++++ salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 2 -- .../pipelines/config/so/9002_output_import.conf.jinja | 2 -- salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja | 2 -- .../logstash/pipelines/config/so/9033_output_snort.conf.jinja | 2 -- .../pipelines/config/so/9034_output_syslog.conf.jinja | 2 -- .../pipelines/config/so/9100_output_osquery.conf.jinja | 2 -- .../pipelines/config/so/9200_output_firewall.conf.jinja | 2 -- .../pipelines/config/so/9400_output_suricata.conf.jinja | 2 -- .../logstash/pipelines/config/so/9500_output_beats.conf.jinja | 2 -- .../logstash/pipelines/config/so/9600_output_ossec.conf.jinja | 2 -- .../pipelines/config/so/9700_output_strelka.conf.jinja | 2 -- 12 files changed, 4 insertions(+), 22 deletions(-) diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index f0de0084f..856f87909 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000 logging.dest: /var/log/kibana/kibana.log telemetry.enabled: false security.showInsecureClusterWarning: false +xpack.security.authc.providers: + anonymous.anonymous1: + order: 0 + credentials: "elasticsearch_anonymous_user" diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index b8e136a52..d17dc2b22 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -12,10 +12,8 @@ output { template_name => "so-zeek" template => "/templates/so-zeek-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index a8acadc89..4562dcee7 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -12,10 +12,8 @@ output { template_name => "so-import" template => "/templates/so-import-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index f2e36b072..fb6eaee5d 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -11,10 +11,8 @@ output { template_name => "so-flow" template => "/templates/so-flow-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index e75ea677a..61aa21a82 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -11,10 +11,8 @@ output { template_name => "so-ids" template => "/templates/so-ids-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 7470cf096..0afbf45ea 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -12,10 +12,8 @@ output { template_name => "so-syslog" template => "/templates/so-syslog-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index b3c2d8e75..3144884d9 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -12,10 +12,8 @@ output { template_name => "so-osquery" template => "/templates/so-osquery-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index e2c75509e..764f597b9 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -11,10 +11,8 @@ output { template_name => "so-firewall" template => "/templates/so-firewall-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index e267cba0a..5013bafc1 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -11,10 +11,8 @@ output { index => "so-ids" template_name => "so-ids" template => "/templates/so-ids-template.json" - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index c4c2d57f6..349c0ada1 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -12,10 +12,8 @@ output { template_name => "so-beats" template => "/templates/so-beats-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 4f55a032b..1a4987a53 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -12,10 +12,8 @@ output { template_name => "so-ossec" template => "/templates/so-ossec-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index a125853ac..d564486e4 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -12,10 +12,8 @@ output { template_name => "so-strelka" template => "/templates/so-strelka-template.json" template_overwrite => true - {%- if grains['role'] in ['so-node','so-heavynode'] %} ssl => true ssl_certificate_verification => false - {%- endif %} } } } From 1ecb079066ed1715946935d4850be5205de4d7b8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 8 Mar 2021 17:36:07 -0500 Subject: [PATCH 15/15] Fix Kibana Script for loading dashboards --- salt/elasticsearch/files/elasticsearch.yml | 6 ------ salt/elasticsearch/init.sls | 2 +- salt/filebeat/init.sls | 1 - salt/kibana/bin/so-kibana-config-load | 11 ++++++++++- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index a83014a1f..fed45bf79 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -9,12 +9,6 @@ {%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %} cluster.name: "{{ ESCLUSTERNAME }}" network.host: 0.0.0.0 - -# minimum_master_nodes need to be explicitly set when bound on a public IP -# set to 1 to allow single node clusters -# Details: https://github.com/elastic/elasticsearch/pull/17288 -#discovery.zen.minimum_master_nodes: 1 -# This is a test -- if this is here, then the volume is mounted correctly. path.logs: /var/log/elasticsearch action.destructive_requires_name: true transport.bind_host: 0.0.0.0 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index bfed1242a..df297986a 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -191,7 +191,7 @@ so-elasticsearch: {% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %} - discovery.type=single-node {% endif %} - - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} + - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true ulimits: - memlock=-1:-1 - nofile=65536:65536 diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 91c6cdbb8..339d307ee 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -13,7 +13,6 @@ # along with this program. If not, see . {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} - {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set LOCALHOSTNAME = salt['grains.get']('host') %} diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index e545dc452..108a4f0f6 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -14,5 +14,14 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson +## This hackery will be removed with Elastic Auth ## + +# Remove the cold cookie if its there +rm /tmp/cookie.txt + +# Let's snag a cookie from Kibana +curl -c /tmp/cookie.txt -X GET http://localhost:5601/ +THECOOKIE=$(cat /tmp/cookie.txt | grep sid | awk '{print $7}') + # Load saved objects -curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1 +curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1