From b8137214e4b4b7f690f8c2a9e8b3fcd05d416e6a Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Fri, 26 Feb 2021 08:08:09 -0500
Subject: [PATCH 01/19] Initial Support - Live Query to Hunt

---
 .../config/so/0008_input_redis.conf.jinja     | 17 ++++++++
 .../config/so/9001_output_osq.conf.jinja      | 41 +++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja
 create mode 100644 salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja

diff --git a/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja
new file mode 100644
index 000000000..694b997bb
--- /dev/null
+++ b/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja
@@ -0,0 +1,17 @@
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
+{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
+
+## TO DO - Add this to Logstash Pipeline Pillar - Manager
+
+input {
+	redis {
+		host => '{{ MANAGER }}'
+		port => 6379
+		data_type => 'pattern_channel'
+		key => 'results_*'
+		type => 'osq'
+		threads => {{ THREADS }}
+		batch_count => {{ BATCH }}
+	}
+}
diff --git a/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja b/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja
new file mode 100644
index 000000000..6a4c564f0
--- /dev/null
+++ b/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja
@@ -0,0 +1,41 @@
+{%- if grains['role'] == 'so-eval' -%}
+{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
+{%- else %}
+{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{%- endif %}
+{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
+
+## TO DO - Add this to Logstash Pipeline Pillar - Search
+
+filter {
+  if [type] =~ "osq" {
+
+ split {
+   field => "rows"
+  }
+ mutate {
+   rename => { 
+      "[rows][cmdline]" => "[process][commandline]"
+      "[rows][name]" => "[process][name]" 
+    }
+    }
+ }
+}
+
+
+output {
+  if [type] =~ "osq" {
+    elasticsearch {
+      pipeline => "common" 
+      hosts => "{{ ES }}"
+      index => "so-osquery"
+      template_name => "so-osquery"
+      template => "/templates/so-osquery-template.json"
+      template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
+    }
+  }
+}

From cf21200a36d6dbd383d82545fd96f476bdcd7b9a Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 2 Mar 2021 09:11:49 -0500
Subject: [PATCH 02/19] Move version to 2.3.31

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 0f1c3e555..dbe139d42 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.3.40
+2.3.31

From 4014dbbc3ddc5a24c82303c98b4edf196d7b3c82 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 2 Mar 2021 10:14:45 -0500
Subject: [PATCH 03/19] Revert "Move version to 2.3.31"

This reverts commit cf21200a36d6dbd383d82545fd96f476bdcd7b9a.
---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index dbe139d42..0f1c3e555 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.3.31
+2.3.40

From 3219f4cd125f2f9235a6ae1fb937b68ab1268fcf Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 2 Mar 2021 11:04:50 -0500
Subject: [PATCH 04/19] Remove Features Option

---
 salt/elasticsearch/files/elasticsearch.yml | 27 ++++++++++------------
 salt/elasticsearch/init.sls                |  9 +-------
 salt/filebeat/init.sls                     |  9 ++------
 salt/kibana/init.sls                       |  8 +------
 salt/logstash/init.sls                     |  9 +-------
 5 files changed, 17 insertions(+), 45 deletions(-)

diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index 25bb6cb02..cdd348f2b 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -1,6 +1,5 @@
 {%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %}
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip') %}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 {%- if TRUECLUSTER is sameas true %}
   {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name') %}
@@ -25,21 +24,19 @@ cluster.routing.allocation.disk.threshold_enabled: true
 cluster.routing.allocation.disk.watermark.low: 95%
 cluster.routing.allocation.disk.watermark.high: 98%
 cluster.routing.allocation.disk.watermark.flood_stage: 98%
-{%- if FEATURES is sameas true %}
 xpack.ml.enabled: false
-#xpack.security.enabled: false
-#xpack.security.transport.ssl.enabled: true
-#xpack.security.transport.ssl.verification_mode: none
-#xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-#xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-#xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
-#{%- if grains['role'] in ['so-node','so-heavynode'] %}
-#xpack.security.http.ssl.enabled: true
-#xpack.security.http.ssl.client_authentication: none
-#xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-#xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-#xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
-#{%- endif %}
+xpack.security.enabled: false
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: none
+xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
+{%- if grains['role'] in ['so-node','so-heavynode'] %}
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.client_authentication: none
+xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
 #xpack.security.authc:
 #  anonymous:
 #    username: anonymous_user
diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls
index 82fc7c77d..debb37512 100644
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -18,17 +18,10 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip') %}
 
-{% if FEATURES is sameas true %}
-  {% set FEATUREZ = "-features" %}
-{% else %}
-  {% set FEATUREZ = '' %}
-{% endif %}
-
 {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %}
   {% set esclustername = salt['pillar.get']('manager:esclustername') %}
   {% set esheap = salt['pillar.get']('manager:esheap') %}
@@ -186,7 +179,7 @@ eslogdir:
 
 so-elasticsearch:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}{{ FEATUREZ }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch
diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index f280309f0..e24c572f6 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -21,12 +21,7 @@
 {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
-{%- if FEATURES is sameas true %}
-  {% set FEATURES = "-features" %}
-{% else %}
-  {% set FEATURES = '' %}
-{% endif %}
+
 filebeatetcdir:
   file.directory:
     - name: /opt/so/conf/filebeat/etc
@@ -64,7 +59,7 @@ filebeatconfsync:
         OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
 so-filebeat:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}{{ FEATURES }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
     - hostname: so-filebeat
     - user: root
     - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }}
diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls
index 10b799e80..fe579ffaa 100644
--- a/salt/kibana/init.sls
+++ b/salt/kibana/init.sls
@@ -4,12 +4,6 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
-{%- if FEATURES is sameas true %}
-  {% set FEATURES = "-features" %}
-{% else %}
-  {% set FEATURES = '' %}
-{% endif %}
 
 # Add ES Group
 kibanasearchgroup:
@@ -73,7 +67,7 @@ kibanabin:
 # Start the kibana docker
 so-kibana:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}{{ FEATURES }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}
     - hostname: kibana
     - user: kibana
     - environment:
diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 61f533865..2c2c89626 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -19,13 +19,6 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
-
-{%- if FEATURES is sameas true %}
-  {% set FEATURES = "-features" %}
-{% else %}
-  {% set FEATURES = '' %}
-{% endif %}
 
 # Logstash Section - Decide which pillar to use
 {% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %}
@@ -146,7 +139,7 @@ lslogdir:
 
 so-logstash:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}{{ FEATURES }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash

From bfd05a8cfc27bd6f43f0951c3d6909c044663e75 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 2 Mar 2021 11:32:29 -0500
Subject: [PATCH 05/19] Change to https for elastic connections

---
 salt/common/tools/sbin/so-features-enable     | 53 -------------------
 salt/common/tools/sbin/soup                   | 10 ----
 salt/elastalert/init.sls                      |  2 +-
 salt/kibana/bin/keepkibanahappy.sh            |  2 +-
 salt/kibana/etc/kibana.yml                    |  3 +-
 .../config/so/9000_output_zeek.conf.jinja     |  1 -
 .../config/so/9002_output_import.conf.jinja   |  1 -
 .../config/so/9004_output_flow.conf.jinja     |  1 -
 .../config/so/9033_output_snort.conf.jinja    |  1 -
 .../config/so/9034_output_syslog.conf.jinja   |  1 -
 .../config/so/9100_output_osquery.conf.jinja  |  1 -
 .../config/so/9200_output_firewall.conf.jinja |  1 -
 .../config/so/9400_output_suricata.conf.jinja |  1 -
 .../config/so/9500_output_beats.conf.jinja    |  1 -
 .../config/so/9600_output_ossec.conf.jinja    |  1 -
 .../config/so/9700_output_strelka.conf.jinja  |  1 -
 salt/soc/files/soc/soc.json                   |  3 +-
 salt/soctopus/files/SOCtopus.conf             |  2 +-
 salt/telegraf/etc/telegraf.conf               |  2 +-
 salt/utility/bin/crossthestreams              |  6 +--
 salt/utility/bin/eval                         |  2 +-
 21 files changed, 10 insertions(+), 86 deletions(-)
 delete mode 100755 salt/common/tools/sbin/so-features-enable

diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable
deleted file mode 100755
index 015b47eba..000000000
--- a/salt/common/tools/sbin/so-features-enable
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
-
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-. /usr/sbin/so-image-common
-local_salt_dir=/opt/so/saltstack/local
-
-cat << EOF
-This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license.
-If you proceed, then we will download new Docker images and restart services.
-
-Please review the Elastic license:
-https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
-
-Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext!
-(We expect to support Elastic Features Security at some point in the future.)
-
-Do you agree to the terms of the Elastic license and understand the note about encryption?
-
-If so, type AGREE to accept the Elastic license and continue.  Otherwise, just press Enter to exit this program without making any changes.
-EOF
-
-read INPUT
-if [ "$INPUT" != "AGREE" ]; then
-        exit
-fi
-
-echo "Please wait while switching to Elastic Features."
-
-require_manager
-
-TRUSTED_CONTAINERS=( \
-  "so-elasticsearch" \
-  "so-filebeat" \
-  "so-kibana" \
-  "so-logstash" )
-update_docker_containers "features" "-features"
-
-# Modify global.sls to enable Features
-sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 6eb01a694..49de2a77a 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -613,16 +613,6 @@ if [ $is_airgap -eq 0 ]; then
 else
   update_registry
   update_docker_containers "soup"
-  FEATURESCHECK=$(lookup_pillar features elastic)
-  if [[ "$FEATURESCHECK" == "True" ]]; then 
-    TRUSTED_CONTAINERS=(
-      "so-elasticsearch"
-      "so-filebeat"
-      "so-kibana"
-      "so-logstash" 
-    )
-    update_docker_containers "features" "-features"
-  fi
 fi
 echo ""
 echo "Stopping Salt Minion service."
diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls
index 9008832f1..d51a77128 100644
--- a/salt/elastalert/init.sls
+++ b/salt/elastalert/init.sls
@@ -104,7 +104,7 @@ elastaconf:
 wait_for_elasticsearch:
   module.run:
     - http.wait_for_successful_query:
-      - url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*'
+      - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*'
       - wait_for: 180
 
 so-elastalert:
diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh
index 541a666bd..5d98ca7a0 100644
--- a/salt/kibana/bin/keepkibanahappy.sh
+++ b/salt/kibana/bin/keepkibanahappy.sh
@@ -4,7 +4,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
+  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml
index 4b9424b4c..ee34e70b1 100644
--- a/salt/kibana/etc/kibana.yml
+++ b/salt/kibana/etc/kibana.yml
@@ -1,11 +1,10 @@
 ---
 # Default Kibana configuration from kibana-docker.
 {%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
 server.name: kibana
 server.host: "0"
 server.basePath: /kibana
-elasticsearch.hosts: [ "http://{{ ES }}:9200" ]
+elasticsearch.hosts: [ "https://{{ ES }}:9200" ]
 #kibana.index: ".kibana"
 #elasticsearch.username: elastic
 #elasticsearch.password: changeme
diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
index d1764eecc..b8e136a52 100644
--- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "zeek" and "import" not in [tags] {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
index 35900471e..a8acadc89 100644
--- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if "import" in [tags] {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
index 0a7d961de..f2e36b072 100644
--- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [event_type] == "sflow" {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
index 7f0e30fbc..e75ea677a 100644
--- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [event_type] == "ids" and "import" not in [tags] {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
index 4a27428f7..7470cf096 100644
--- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "syslog" {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
index 2a71e3fab..b3c2d8e75 100644
--- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "osquery" {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
index 9b93d327b..e2c75509e 100644
--- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [dataset] =~ "firewall" {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
index 29837040a..e267cba0a 100644
--- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "suricata" and "import" not in [tags] {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
index beaf24727..c4c2d57f6 100644
--- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if "beat-ext" in [tags] and "import" not in [tags] {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
index 95c81577b..4f55a032b 100644
--- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "ossec" {
     elasticsearch {
diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
index 1e8c44cc6..a125853ac 100644
--- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "strelka" {
     elasticsearch {
diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index bda2df431..cf90e5a9d 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -1,7 +1,6 @@
 {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
 {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %}
 {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
 {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %}
 {%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %}
@@ -31,7 +30,7 @@
         "hostUrl": "http://{{ MANAGERIP }}:4434/"
       },
       "elastic": {
-        "hostUrl": "http://{{ MANAGERIP }}:9200",
+        "hostUrl": "https://{{ MANAGERIP }}:9200",
         {%- if salt['pillar.get']('nodestab', {}) %}
         "remoteHostUrls": [
         {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf
index 4f58ecf83..c9941c3e1 100644
--- a/salt/soctopus/files/SOCtopus.conf
+++ b/salt/soctopus/files/SOCtopus.conf
@@ -6,7 +6,7 @@
 
 
 [es]
-es_url = http://{{MANAGER}}:9200
+es_url = https://{{MANAGER}}:9200
 es_ip = {{MANAGER}}
 es_user = YOURESUSER
 es_pass = YOURESPASS
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index 0c447172f..b8976b8c9 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -622,7 +622,7 @@
 #   ## specify a list of one or more Elasticsearch servers
 #   # you can add username and password to your url to use basic authentication:
 #   # servers = ["http://user:pass@localhost:9200"]
-   servers = ["http://{{ MANAGER }}:9200"]
+   servers = ["https://{{ MANAGER }}:9200"]
 {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode']  %}
  [[inputs.elasticsearch]]
    servers = ["https://{{ NODEIP }}:9200"]
diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams
index 490c7b548..5fed02fa9 100644
--- a/salt/utility/bin/crossthestreams
+++ b/salt/utility/bin/crossthestreams
@@ -9,7 +9,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
+  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
@@ -29,7 +29,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 
 echo "Applying cross cluster search config..."
-    curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+    curl -s -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
          -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
 
@@ -37,7 +37,7 @@ echo "Applying cross cluster search config..."
 {%- if TRUECLUSTER is sameas false %}
   {%- if salt['pillar.get']('nodestab', {}) %}
     {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-curl -XPUT -L http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
+curl -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
     {%- endfor %}
   {%- endif %}
 {%- endif %}
diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval
index f63a61942..c4af1cfbc 100644
--- a/salt/utility/bin/eval
+++ b/salt/utility/bin/eval
@@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
+  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"

From 80574d3c202fe46850af043d1184bb322c692102 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 2 Mar 2021 13:59:43 -0500
Subject: [PATCH 06/19] Make https default for all things

---
 salt/elasticsearch/files/elasticsearch.yml |  2 --
 salt/elasticsearch/files/sotls.yml         | 17 -----------------
 2 files changed, 19 deletions(-)
 delete mode 100644 salt/elasticsearch/files/sotls.yml

diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index cdd348f2b..bf9f60c60 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -31,7 +31,6 @@ xpack.security.transport.ssl.verification_mode: none
 xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
 xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
 xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
 xpack.security.http.ssl.enabled: true
 xpack.security.http.ssl.client_authentication: none
 xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
@@ -42,7 +41,6 @@ xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config
 #    username: anonymous_user
 #    roles: superuser
 #    authz_exception: true
-{%- endif %}
 node.name: {{ grains.host }}
 script.max_compilations_rate: 1000/1m
 {%- if TRUECLUSTER is sameas true %}
diff --git a/salt/elasticsearch/files/sotls.yml b/salt/elasticsearch/files/sotls.yml
deleted file mode 100644
index 2fc577337..000000000
--- a/salt/elasticsearch/files/sotls.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-keystore.path: /usr/share/elasticsearch/config/sokeys
-keystore.password: changeit 
-keystore.algorithm: SunX509
-truststore.path: /etc/pki/java/cacerts
-truststore.password: changeit
-truststore.algorithm: PKIX
-protocols:
-- TLSv1.2
-ciphers:
-- TLS_RSA_WITH_AES_128_CBC_SHA256
-- TLS_RSA_WITH_AES_256_GCM_SHA384
-transport.encrypted: true
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
-http.encrypted: true
-{%- else %}
-http.encrypted: false
-{%- endif %}

From e0d9212e55ebbf9814b0a8a05893b8ae875fbe88 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 2 Mar 2021 14:01:05 -0500
Subject: [PATCH 07/19] Make https default for all things

---
 salt/elasticsearch/init.sls | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls
index debb37512..bfed1242a 100644
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -140,14 +140,6 @@ esyml:
     - group: 939
     - template: jinja
 
-sotls:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/sotls.yml
-    - source: salt://elasticsearch/files/sotls.yml
-    - user: 930
-    - group: 939
-    - template: jinja
-
 #sync templates to /opt/so/conf/elasticsearch/templates
 {% for TEMPLATE in TEMPLATES %}
 es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
@@ -221,7 +213,6 @@ so-elasticsearch:
       - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
       - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
       - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
-      - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro
     - watch:
       - file: cacertz
       - file: esyml

From 4f867e53756a382e7d09bcfceb247a9901718f1f Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 10:02:23 -0500
Subject: [PATCH 08/19] Fix all scripts for ssl elastic

---
 salt/common/tools/sbin/so-elastic-clear              | 12 ------------
 salt/common/tools/sbin/so-elasticsearch-indices-rw   |  5 ++---
 .../tools/sbin/so-elasticsearch-pipeline-stats       |  8 --------
 .../tools/sbin/so-elasticsearch-pipelines-list       |  8 --------
 .../tools/sbin/so-elasticsearch-templates-list       |  8 --------
 .../tools/sbin/so-elasticsearch-templates-load       |  8 --------
 salt/common/tools/sbin/so-index-list                 |  4 ----
 salt/curator/files/curator.yml                       |  4 ++--
 salt/elasticsearch/files/so-elasticsearch-pipelines  | 10 +---------
 salt/filebeat/etc/filebeat.yml                       |  2 +-
 salt/utility/bin/crossthestreams                     |  7 +++----
 salt/utility/bin/eval                                |  4 ++--
 12 files changed, 11 insertions(+), 69 deletions(-)

diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear
index 78e80d014..4c7271272 100755
--- a/salt/common/tools/sbin/so-elastic-clear
+++ b/salt/common/tools/sbin/so-elastic-clear
@@ -50,11 +50,7 @@ done
 if [ $SKIP -ne 1 ]; then
         # List indices
         echo 
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        {% else %}
-        curl -L {{ NODEIP }}:9200/_cat/indices?v
-        {% endif %}
         echo
         # Inform user we are about to delete all data
         echo
@@ -93,18 +89,10 @@ fi
 # Delete data
 echo "Deleting data..."
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
 for INDX in ${INDXS}
 do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% else %}
-    curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% endif %}
 done
 
 #Start Logstash/Filebeat
diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw
index 837b22090..a858db51e 100755
--- a/salt/common/tools/sbin/so-elasticsearch-indices-rw
+++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw
@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
-for p in $ESPORT $THEHIVEESPORT; do
-   curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-done
+curl -s -k -XPUT -H "Content-Type: application/json" -L http://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
index 234be2017..146196917 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
+++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
@@ -19,15 +19,7 @@
 . /usr/sbin/so-common
 
 if [ "$1" == "" ]; then
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% endif %}
 else
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% endif %}
 fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
index 37da36127..565f90071 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list
+++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% endif %}
 fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list
index aa011b573..494ca5770 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-list
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-list
@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
-    {% endif %}
 fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load
index 76558e17a..42a836854 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-load
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-load
@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-    {% else %}
-	curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
 
 
 echo "Loading templates..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% else %}
-for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% endif %}
 echo
 
 cd - >/dev/null
diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list
index 1ed27a095..dcfebbf58 100755
--- a/salt/common/tools/sbin/so-index-list
+++ b/salt/common/tools/sbin/so-index-list
@@ -15,8 +15,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 curl -X GET -k -L https://localhost:9200/_cat/indices?v
-{% else %}
-curl -X GET -L localhost:9200/_cat/indices?v
-{% endif %}
diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml
index 016a123f0..7d86ccc04 100644
--- a/salt/curator/files/curator.yml
+++ b/salt/curator/files/curator.yml
@@ -12,11 +12,11 @@ client:
     - {{elasticsearch}}
   port: 9200
   url_prefix:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  use_ssl: True{% else %}  use_ssl: False{% endif %}
+  use_ssl: True
   certificate:
   client_cert:
   client_key:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  ssl_no_validate: True{% else %}  ssl_no_validate: False{% endif %}
+  ssl_no_validate: True
   http_auth:
   timeout: 30
   master_only: False
diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines
index dce6a081b..fca50b7d4 100755
--- a/salt/elasticsearch/files/so-elasticsearch-pipelines
+++ b/salt/elasticsearch/files/so-elasticsearch-pipelines
@@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
-	curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% else %}
-	curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
+    curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -51,11 +47,7 @@ fi
 cd ${ELASTICSEARCH_INGEST_PIPELINES}
 
 echo "Loading pipelines..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% else %}
-for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% endif %}
 echo
 
 cd - >/dev/null
diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 1b902d542..b6aa218ef 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -260,7 +260,7 @@ output.{{ type }}:
   {%- if grains['role'] in ["so-eval", "so-import"] %}
 output.elasticsearch:
   enabled: true
-  hosts: ["{{ MANAGER }}:9200"]
+  hosts: ["https://{{ MANAGER }}:9200"]
   pipelines:
     - pipeline: "%{[module]}.%{[dataset]}"
   indices:
diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams
index 5fed02fa9..3838f67df 100644
--- a/salt/utility/bin/crossthestreams
+++ b/salt/utility/bin/crossthestreams
@@ -1,7 +1,6 @@
 #!/bin/bash
 {% set ES = salt['pillar.get']('manager:mainip', '') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 
 # Wait for ElasticSearch to come up, so that we can query for version infromation
@@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
+  curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
@@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 
 echo "Applying cross cluster search config..."
-    curl -s -XPUT -L https://{{ ES }}:9200/_cluster/settings \
+    curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
          -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
 
@@ -37,7 +36,7 @@ echo "Applying cross cluster search config..."
 {%- if TRUECLUSTER is sameas false %}
   {%- if salt['pillar.get']('nodestab', {}) %}
     {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-curl -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
+curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
     {%- endfor %}
   {%- endif %}
 {%- endif %}
diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval
index c4af1cfbc..dcf46de7a 100644
--- a/salt/utility/bin/eval
+++ b/salt/utility/bin/eval
@@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
+  curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
   if [ $? -eq 0 ]; then
     ELASTICSEARCH_CONNECTED="yes"
     echo "connected!"
@@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 
 echo "Applying cross cluster search config..."
-    curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+    curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
          -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"

From 07b5cc3d1d24342db04074d01ef8d1425505bb19 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 10:29:41 -0500
Subject: [PATCH 09/19] Fix https for rw indicies script

---
 salt/common/tools/sbin/so-elasticsearch-indices-rw | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw
index a858db51e..6b123bd0d 100755
--- a/salt/common/tools/sbin/so-elasticsearch-indices-rw
+++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw
@@ -21,5 +21,5 @@ THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
-curl -s -k -XPUT -H "Content-Type: application/json" -L http://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
+curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
 curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

From 4f34eca5b94336ca44d62ee4b1ed6e8e90a1fc31 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 10:32:23 -0500
Subject: [PATCH 10/19] remove unused script

---
 salt/kibana/bin/keepkibanahappy.sh | 53 ------------------------------
 1 file changed, 53 deletions(-)
 delete mode 100644 salt/kibana/bin/keepkibanahappy.sh

diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh
deleted file mode 100644
index 5d98ca7a0..000000000
--- a/salt/kibana/bin/keepkibanahappy.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-# Wait for ElasticSearch to come up, so that we can query for version infromation
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
-  if [ $? -eq 0 ]; then
-    ELASTICSEARCH_CONNECTED="yes"
-    echo "connected!"
-    break
-  else
-    ((COUNT+=1))
-    sleep 1
-    echo -n "."
-  fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-  echo
-  echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-  echo
-
-  exit
-fi
-
-# Make sure Kibana is running
-MAX_WAIT=240
-
-# Check to see if Kibana is available
-wait_step=0
-  until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do
-  wait_step=$(( ${wait_step} + 1 ))
-  echo "Waiting on Kibana...Attempt #$wait_step"
-	  if [ ${wait_step} -gt ${MAX_WAIT} ]; then
-			  echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
-			  exit 5
-	  fi
-		  sleep 1s;
-  done
-
-
-# Apply Kibana template
-  echo
-  echo "Applying Kibana template..."
-  curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \
-       -H 'Content-Type: application/json' \
-       -d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}'
-  echo
-
-  curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \
-       -H 'Content-Type: application/json' \
-       -d'{"index" : {"number_of_replicas" : 0}}'
-  echo

From 4de62c878ccb26e470a667fb042c54a1fb394594 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 12:51:29 -0500
Subject: [PATCH 11/19] turn on elastic security

---
 salt/elasticsearch/files/elasticsearch.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index bf9f60c60..a83014a1f 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -25,7 +25,7 @@ cluster.routing.allocation.disk.watermark.low: 95%
 cluster.routing.allocation.disk.watermark.high: 98%
 cluster.routing.allocation.disk.watermark.flood_stage: 98%
 xpack.ml.enabled: false
-xpack.security.enabled: false
+xpack.security.enabled: true
 xpack.security.transport.ssl.enabled: true
 xpack.security.transport.ssl.verification_mode: none
 xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
@@ -36,11 +36,11 @@ xpack.security.http.ssl.client_authentication: none
 xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
 xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
 xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
-#xpack.security.authc:
-#  anonymous:
-#    username: anonymous_user
-#    roles: superuser
-#    authz_exception: true
+xpack.security.authc:
+  anonymous:
+    username: anonymous_user
+    roles: superuser
+    authz_exception: true
 node.name: {{ grains.host }}
 script.max_compilations_rate: 1000/1m
 {%- if TRUECLUSTER is sameas true %}

From 6c7111cd0aec19ce57f555e14ac9d24b4ede3d90 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 13:42:04 -0500
Subject: [PATCH 12/19] turn off verification mode for ES

---
 salt/kibana/etc/kibana.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml
index ee34e70b1..f0de0084f 100644
--- a/salt/kibana/etc/kibana.yml
+++ b/salt/kibana/etc/kibana.yml
@@ -5,6 +5,7 @@ server.name: kibana
 server.host: "0"
 server.basePath: /kibana
 elasticsearch.hosts: [ "https://{{ ES }}:9200" ]
+elasticsearch.ssl.verificationMode: none
 #kibana.index: ".kibana"
 #elasticsearch.username: elastic
 #elasticsearch.password: changeme

From e0538417f1cec9f280f367d4907098164ee392bc Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 14:06:35 -0500
Subject: [PATCH 13/19] fix http.wait

---
 salt/elastalert/init.sls | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls
index d51a77128..8fcb46cda 100644
--- a/salt/elastalert/init.sls
+++ b/salt/elastalert/init.sls
@@ -106,6 +106,7 @@ wait_for_elasticsearch:
     - http.wait_for_successful_query:
       - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*'
       - wait_for: 180
+      - verify_ssl: False
 
 so-elastalert:
   docker_container.running:

From bf4249d28b0f28450a14aec29df7227e92cfd0b7 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 14:16:10 -0500
Subject: [PATCH 14/19] fix elastalert verification

---
 salt/elastalert/defaults.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml
index a22d65b7b..ad675b8ee 100644
--- a/salt/elastalert/defaults.yaml
+++ b/salt/elastalert/defaults.yaml
@@ -16,8 +16,8 @@ elastalert:
     #aws_region: us-east-1
     #profile: test
     #es_url_prefix: elasticsearch
-    #use_ssl: True
-    #verify_certs: True
+    use_ssl: true
+    verify_certs: false
     #es_send_get_body_as: GET
     #es_username: someusername
     #es_password: somepassword

From 49371a1d6a09aa301f2307abdcdd0ce56a574379 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Mar 2021 14:30:45 -0500
Subject: [PATCH 15/19] fix elastic output for ssl

---
 salt/kibana/etc/kibana.yml                                    | 4 ++++
 salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 2 --
 .../pipelines/config/so/9002_output_import.conf.jinja         | 2 --
 salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja | 2 --
 .../logstash/pipelines/config/so/9033_output_snort.conf.jinja | 2 --
 .../pipelines/config/so/9034_output_syslog.conf.jinja         | 2 --
 .../pipelines/config/so/9100_output_osquery.conf.jinja        | 2 --
 .../pipelines/config/so/9200_output_firewall.conf.jinja       | 2 --
 .../pipelines/config/so/9400_output_suricata.conf.jinja       | 2 --
 .../logstash/pipelines/config/so/9500_output_beats.conf.jinja | 2 --
 .../logstash/pipelines/config/so/9600_output_ossec.conf.jinja | 2 --
 .../pipelines/config/so/9700_output_strelka.conf.jinja        | 2 --
 12 files changed, 4 insertions(+), 22 deletions(-)

diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml
index f0de0084f..856f87909 100644
--- a/salt/kibana/etc/kibana.yml
+++ b/salt/kibana/etc/kibana.yml
@@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000
 logging.dest: /var/log/kibana/kibana.log
 telemetry.enabled: false
 security.showInsecureClusterWarning: false
+xpack.security.authc.providers:
+  anonymous.anonymous1:
+    order: 0
+    credentials: "elasticsearch_anonymous_user" 
diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
index b8e136a52..d17dc2b22 100644
--- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
@@ -12,10 +12,8 @@ output {
       template_name => "so-zeek"
       template => "/templates/so-zeek-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
index a8acadc89..4562dcee7 100644
--- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
@@ -12,10 +12,8 @@ output {
       template_name => "so-import"
       template => "/templates/so-import-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
index f2e36b072..fb6eaee5d 100644
--- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
@@ -11,10 +11,8 @@ output {
       template_name => "so-flow"
       template => "/templates/so-flow-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
index e75ea677a..61aa21a82 100644
--- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
@@ -11,10 +11,8 @@ output {
       template_name => "so-ids"
       template => "/templates/so-ids-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
index 7470cf096..0afbf45ea 100644
--- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
@@ -12,10 +12,8 @@ output {
       template_name => "so-syslog"
       template => "/templates/so-syslog-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
index b3c2d8e75..3144884d9 100644
--- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
@@ -12,10 +12,8 @@ output {
       template_name => "so-osquery"
       template => "/templates/so-osquery-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
index e2c75509e..764f597b9 100644
--- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
@@ -11,10 +11,8 @@ output {
       template_name => "so-firewall"
       template => "/templates/so-firewall-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
index e267cba0a..5013bafc1 100644
--- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
@@ -11,10 +11,8 @@ output {
       index => "so-ids"
       template_name => "so-ids" 
       template => "/templates/so-ids-template.json"
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
index c4c2d57f6..349c0ada1 100644
--- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
@@ -12,10 +12,8 @@ output {
       template_name => "so-beats"
       template => "/templates/so-beats-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
index 4f55a032b..1a4987a53 100644
--- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
@@ -12,10 +12,8 @@ output {
       template_name => "so-ossec"
       template => "/templates/so-ossec-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }
diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
index a125853ac..d564486e4 100644
--- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
@@ -12,10 +12,8 @@ output {
       template_name => "so-strelka"
       template => "/templates/so-strelka-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

From 548f67ca6f8a29c94240a82cf99464dc28e2b6d6 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Thu, 4 Mar 2021 18:21:13 -0500
Subject: [PATCH 16/19] Initial support for Live Queries in Hunt

---
 pillar/logstash/manager.sls                   |  1 +
 pillar/logstash/search.sls                    |  1 +
 .../files/ingest/osquery.live_query           | 16 +++++++++++++
 .../files/ingest/osquery.normalize            | 14 +++++++++++
 .../files/ingest/osquery.query_result         | 17 +++++--------
 .../templates/so/so-common-template.json      |  4 ++++
 ... => 0008_input_fleet_livequery.conf.jinja} |  8 ++++---
 .../config/so/9100_output_osquery.conf.jinja  |  2 +-
 ... 9101_output_osquery_livequery.conf.jinja} | 24 ++++++++++---------
 salt/soc/files/soc/hunt.queries.json          |  1 +
 10 files changed, 62 insertions(+), 26 deletions(-)
 create mode 100644 salt/elasticsearch/files/ingest/osquery.live_query
 create mode 100644 salt/elasticsearch/files/ingest/osquery.normalize
 rename salt/logstash/pipelines/config/so/{0008_input_redis.conf.jinja => 0008_input_fleet_livequery.conf.jinja} (74%)
 rename salt/logstash/pipelines/config/so/{9001_output_osq.conf.jinja => 9101_output_osquery_livequery.conf.jinja} (72%)

diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls
index 6f3ba495b..1aa445c97 100644
--- a/pillar/logstash/manager.sls
+++ b/pillar/logstash/manager.sls
@@ -3,6 +3,7 @@ logstash:
   pipelines:
     manager:
       config:
+        - so/0008_input_fleet_livequery.conf.jinja
         - so/0009_input_beats.conf      
         - so/0010_input_hhbeats.conf
         - so/9999_output_redis.conf.jinja
diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls
index 7a5aeec39..8ae84fe1f 100644
--- a/pillar/logstash/search.sls
+++ b/pillar/logstash/search.sls
@@ -8,6 +8,7 @@ logstash:
         - so/9002_output_import.conf.jinja
         - so/9034_output_syslog.conf.jinja
         - so/9100_output_osquery.conf.jinja
+        - so/9101_output_osquery_livequery.conf.jinja        
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja
diff --git a/salt/elasticsearch/files/ingest/osquery.live_query b/salt/elasticsearch/files/ingest/osquery.live_query
new file mode 100644
index 000000000..92f325e1d
--- /dev/null
+++ b/salt/elasticsearch/files/ingest/osquery.live_query
@@ -0,0 +1,16 @@
+{
+  "description" : "osquery live query",
+  "processors" : [
+    {
+      "script": {
+        "lang": "painless",
+        "source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; "
+      }
+    },
+    { "remove":      { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } },
+    { "rename":      { "field": "distributed_query_execution_id",  "target_field": "result.query_id",       "ignore_missing": true  } },   
+    { "rename":      { "field": "computer_name",  "target_field": "host.hostname",       "ignore_missing": true  } },        
+    { "pipeline":    { "name": "osquery.normalize" } },
+    { "pipeline":    { "name": "common" } }
+  ]
+}
diff --git a/salt/elasticsearch/files/ingest/osquery.normalize b/salt/elasticsearch/files/ingest/osquery.normalize
new file mode 100644
index 000000000..ce0a6ca92
--- /dev/null
+++ b/salt/elasticsearch/files/ingest/osquery.normalize
@@ -0,0 +1,14 @@
+{
+  "description" : "osquery normalize",
+  "processors" : [
+     { "rename":      { "field": "result.columns.cmdline",               "target_field": "process.command_line",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.cwd",               "target_field": "process.working_directory",       "ignore_missing": true  } },   
+     { "rename":      { "field": "result.columns.name",               "target_field": "process.name",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.path",               "target_field": "process.executable",       "ignore_missing": true  } }, 
+     { "rename":      { "field": "result.columns.pid",               "target_field": "process.pid",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.parent",               "target_field": "process.ppid",       "ignore_missing": true  } }, 
+     { "rename":      { "field": "result.columns.uid",               "target_field": "user.id",       "ignore_missing": true  } },   
+     { "rename":      { "field": "result.columns.username",           "target_field": "user.name",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.gid",               "target_field": "group.id",       "ignore_missing": true  } }     
+  ]
+}
diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result
index b6b4f22ef..9bb381946 100644
--- a/salt/elasticsearch/files/ingest/osquery.query_result
+++ b/salt/elasticsearch/files/ingest/osquery.query_result
@@ -1,24 +1,19 @@
 {
   "description" : "osquery",
   "processors" : [
-    { "json":        { "field": "message",                                            "target_field": "message2",             "ignore_failure": true  } },
-    { "gsub":        { "field": "message2.columns.data",                              "pattern": "\\\\xC2\\\\xAE",             "replacement": "", "ignore_missing": true  } },
-    { "rename":      { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns",  "target_field": "winlog",       "ignore_missing": true  } },
+    { "json":        { "field": "message",                                            "target_field": "result",             "ignore_failure": true  } },
+    { "gsub":        { "field": "result.columns.data",                              "pattern": "\\\\xC2\\\\xAE",             "replacement": "", "ignore_missing": true  } },
+    { "rename":      { "if": "ctx.result.columns?.eventid != null", "field": "result.columns",  "target_field": "winlog",       "ignore_missing": true  } },
     { "json":        { "field": "winlog.data",    "target_field": "unparsed",      "ignore_failure": true} },
     { "set":         { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true  } },     
     { "rename":      { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data",       "ignore_missing": true, "ignore_failure": true  } },   
     { "rename":      { "field": "winlog.source",               "target_field": "winlog.channel",       "ignore_missing": true  } },    
     { "rename":      { "field": "winlog.eventid",               "target_field": "winlog.event_id",       "ignore_missing": true  } },   
     { "pipeline":    { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
-    { "pipeline":    { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
-    {
-      "script": {
-        "lang": "painless",
-        "source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; "
-      }
-    },
+    { "pipeline":    { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')",  "name":"win.eventlogs" }  },
     { "set":           { "field": "event.module", "value": "osquery", "override": false }  },
-    { "set":           { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false}  },
+    { "set":           { "field": "event.dataset", "value": "{{result.name}}", "override": false}  },
+    { "pipeline":    { "if": "!(ctx.containsKey('winlog'))",  "name": "osquery.normalize" } },
     { "pipeline":    { "name": "common" } }
   ]
 }
diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json
index 062838670..012c590d0 100644
--- a/salt/elasticsearch/templates/so/so-common-template.json
+++ b/salt/elasticsearch/templates/so/so-common-template.json
@@ -365,6 +365,10 @@
 	      "request":{
           "type":"object",
           "dynamic": true
+        },
+	      "result":{
+          "type":"object",
+          "dynamic": true
         },
         "rfb":{
           "type":"object",
diff --git a/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja
similarity index 74%
rename from salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja
rename to salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja
index 694b997bb..83aa0c02d 100644
--- a/salt/logstash/pipelines/config/so/0008_input_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja
@@ -2,15 +2,17 @@
 {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
 {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
 
-## TO DO - Add this to Logstash Pipeline Pillar - Manager
-
 input {
 	redis {
 		host => '{{ MANAGER }}'
 		port => 6379
 		data_type => 'pattern_channel'
 		key => 'results_*'
-		type => 'osq'
+		type => 'live_query'
+        add_field => {
+        "module" => "osquery"
+          "dataset" => "live_query"
+          }        
 		threads => {{ THREADS }}
 		batch_count => {{ BATCH }}
 	}
diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
index 2a71e3fab..9e660f8a8 100644
--- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
@@ -5,7 +5,7 @@
 {%- endif %}
 {% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
-  if [module] =~ "osquery" {
+  if [module] =~ "osquery" and "live_query" not in [dataset] {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
diff --git a/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja
similarity index 72%
rename from salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja
rename to salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja
index 6a4c564f0..51e691176 100644
--- a/salt/logstash/pipelines/config/so/9001_output_osq.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja
@@ -5,28 +5,30 @@
 {%- endif %}
 {% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 
-## TO DO - Add this to Logstash Pipeline Pillar - Search
-
 filter {
-  if [type] =~ "osq" {
+  if [type] =~ "live_query" {
 
+    mutate {
+        rename => {
+            "[host][hostname]" => "computer_name"
+        }
+    }
+
+ prune {
+        blacklist_names => ["host"]
+      }
+    
  split {
    field => "rows"
   }
- mutate {
-   rename => { 
-      "[rows][cmdline]" => "[process][commandline]"
-      "[rows][name]" => "[process][name]" 
-    }
-    }
  }
 }
 
 
 output {
-  if [type] =~ "osq" {
+  if [type] =~ "live_query" {
     elasticsearch {
-      pipeline => "common" 
+      pipeline => "osquery.live_query" 
       hosts => "{{ ES }}"
       index => "so-osquery"
       template_name => "so-osquery"
diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json
index b8dc5eb21..5f3a359b5 100644
--- a/salt/soc/files/soc/hunt.queries.json
+++ b/salt/soc/files/soc/hunt.queries.json
@@ -42,6 +42,7 @@
           { "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.dataset:mysql | groupby mysql.command"},
           { "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.dataset:notice | groupby notice.note notice.message"},
           { "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.dataset:ntlm | groupby ntlm.server.dns.name"},
+          { "name": "Osquery Live Queries", "description": "Osquery Live Query results grouped by computer name", "query": "event.dataset:live_query | groupby host.hostname"},
           { "name": "PE", "description": "PE files list", "query": "event.dataset:pe | groupby file.machine file.os file.subsystem"},
           { "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.dataset:radius | groupby user.name.keyword"},
           { "name": "RDP", "description": "RDP grouped by client name", "query": "event.dataset:rdp | groupby client.name"},

From 84c152e233a458d6f7cc91850e04751fac904c8d Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 8 Mar 2021 12:05:44 -0500
Subject: [PATCH 17/19] fix: Sensors can temporarily show offline while
 processing large PCAP jobs. Resolves #3279.

---
 salt/soc/files/soc/soc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index bda2df431..0742eb33e 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -45,7 +45,7 @@
       },
       "sostatus": {
         "refreshIntervalMs": 30000,
-        "offlineThresholdMs": 60000
+        "offlineThresholdMs": 900000
       },
 {% if THEHIVEKEY != '' %}
       "thehive": {

From 1ecb079066ed1715946935d4850be5205de4d7b8 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 8 Mar 2021 17:36:07 -0500
Subject: [PATCH 18/19] Fix Kibana Script for loading dashboards

---
 salt/elasticsearch/files/elasticsearch.yml |  6 ------
 salt/elasticsearch/init.sls                |  2 +-
 salt/filebeat/init.sls                     |  1 -
 salt/kibana/bin/so-kibana-config-load      | 11 ++++++++++-
 4 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index a83014a1f..fed45bf79 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -9,12 +9,6 @@
 {%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %}
 cluster.name: "{{ ESCLUSTERNAME }}"
 network.host: 0.0.0.0
-
-# minimum_master_nodes need to be explicitly set when bound on a public IP
-# set to 1 to allow single node clusters
-# Details: https://github.com/elastic/elasticsearch/pull/17288
-#discovery.zen.minimum_master_nodes: 1
-# This is a test -- if this is here, then the volume is mounted correctly.
 path.logs: /var/log/elasticsearch
 action.destructive_requires_name: true
 transport.bind_host: 0.0.0.0
diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls
index bfed1242a..df297986a 100644
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -191,7 +191,7 @@ so-elasticsearch:
       {% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
       - discovery.type=single-node
       {% endif %}
-      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }}
+      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true
       ulimits:
       - memlock=-1:-1
       - nofile=65536:65536
diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index 91c6cdbb8..339d307ee 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -13,7 +13,6 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
-
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set LOCALHOSTNAME = salt['grains.get']('host') %}
diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load
index e545dc452..108a4f0f6 100644
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -14,5 +14,14 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o
 # SOCtopus and Manager
 sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson
 
+## This hackery will be removed with Elastic Auth ##
+
+# Remove the cold cookie if its there
+rm /tmp/cookie.txt
+
+# Let's snag a cookie from Kibana
+curl -c /tmp/cookie.txt -X GET http://localhost:5601/
+THECOOKIE=$(cat /tmp/cookie.txt | grep sid | awk '{print $7}')
+
 # Load saved objects
-curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
+curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1

From 49258a13a39b07e70135a459ac54df1eb4607f51 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 9 Mar 2021 10:03:29 -0500
Subject: [PATCH 19/19] Make saved objects less hacky

---
 salt/kibana/bin/so-kibana-config-load | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load
index 108a4f0f6..b40227031 100644
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -14,14 +14,9 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o
 # SOCtopus and Manager
 sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson
 
-## This hackery will be removed with Elastic Auth ##
-
-# Remove the cold cookie if its there
-rm /tmp/cookie.txt
+## This hackery will be removed if using Elastic Auth ##
 
 # Let's snag a cookie from Kibana
-curl -c /tmp/cookie.txt -X GET http://localhost:5601/
-THECOOKIE=$(cat /tmp/cookie.txt | grep sid | awk '{print $7}')
-
+THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
 # Load saved objects
 curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1