diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls
index 6f3ba495b..1aa445c97 100644
--- a/pillar/logstash/manager.sls
+++ b/pillar/logstash/manager.sls
@@ -3,6 +3,7 @@ logstash:
pipelines:
manager:
config:
+ - so/0008_input_fleet_livequery.conf.jinja
- so/0009_input_beats.conf
- so/0010_input_hhbeats.conf
- so/9999_output_redis.conf.jinja
diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls
index 7a5aeec39..8ae84fe1f 100644
--- a/pillar/logstash/search.sls
+++ b/pillar/logstash/search.sls
@@ -8,6 +8,7 @@ logstash:
- so/9002_output_import.conf.jinja
- so/9034_output_syslog.conf.jinja
- so/9100_output_osquery.conf.jinja
+ - so/9101_output_osquery_livequery.conf.jinja
- so/9400_output_suricata.conf.jinja
- so/9500_output_beats.conf.jinja
- so/9600_output_ossec.conf.jinja
diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear
index 78e80d014..4c7271272 100755
--- a/salt/common/tools/sbin/so-elastic-clear
+++ b/salt/common/tools/sbin/so-elastic-clear
@@ -50,11 +50,7 @@ done
if [ $SKIP -ne 1 ]; then
# List indices
echo
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
- {% else %}
- curl -L {{ NODEIP }}:9200/_cat/indices?v
- {% endif %}
echo
# Inform user we are about to delete all data
echo
@@ -93,18 +89,10 @@ fi
# Delete data
echo "Deleting data..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
for INDX in ${INDXS}
do
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
- {% else %}
- curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
- {% endif %}
done
#Start Logstash/Filebeat
diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw
index 837b22090..6b123bd0d 100755
--- a/salt/common/tools/sbin/so-elasticsearch-indices-rw
+++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw
@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
echo "Removing read only attributes for indices..."
echo
-for p in $ESPORT $THEHIVEESPORT; do
- curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
-done
+curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
index 234be2017..146196917 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
+++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats
@@ -19,15 +19,7 @@
. /usr/sbin/so-common
if [ "$1" == "" ]; then
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
- {% endif %}
else
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
- {% endif %}
fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
index 37da36127..565f90071 100755
--- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list
+++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list
@@ -17,15 +17,7 @@
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
. /usr/sbin/so-common
if [ "$1" == "" ]; then
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
- {% endif %}
else
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
- {% endif %}
fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list
index aa011b573..494ca5770 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-list
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-list
@@ -17,15 +17,7 @@
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
. /usr/sbin/so-common
if [ "$1" == "" ]; then
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
- {% endif %}
else
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
- {% else %}
- curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
- {% endif %}
fi
diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load
index 76558e17a..42a836854 100755
--- a/salt/common/tools/sbin/so-elasticsearch-templates-load
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-load
@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
- {% if grains['role'] in ['so-node','so-heavynode'] %}
curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% else %}
- curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% endif %}
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
echo "Loading templates..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% else %}
-for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% endif %}
echo
cd - >/dev/null
diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable
deleted file mode 100755
index 015b47eba..000000000
--- a/salt/common/tools/sbin/so-features-enable
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-. /usr/sbin/so-common
-. /usr/sbin/so-image-common
-local_salt_dir=/opt/so/saltstack/local
-
-cat << EOF
-This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license.
-If you proceed, then we will download new Docker images and restart services.
-
-Please review the Elastic license:
-https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
-
-Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext!
-(We expect to support Elastic Features Security at some point in the future.)
-
-Do you agree to the terms of the Elastic license and understand the note about encryption?
-
-If so, type AGREE to accept the Elastic license and continue. Otherwise, just press Enter to exit this program without making any changes.
-EOF
-
-read INPUT
-if [ "$INPUT" != "AGREE" ]; then
- exit
-fi
-
-echo "Please wait while switching to Elastic Features."
-
-require_manager
-
-TRUSTED_CONTAINERS=( \
- "so-elasticsearch" \
- "so-filebeat" \
- "so-kibana" \
- "so-logstash" )
-update_docker_containers "features" "-features"
-
-# Modify global.sls to enable Features
-sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls
diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list
index 1ed27a095..dcfebbf58 100755
--- a/salt/common/tools/sbin/so-index-list
+++ b/salt/common/tools/sbin/so-index-list
@@ -15,8 +15,4 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -X GET -k -L https://localhost:9200/_cat/indices?v
-{% else %}
-curl -X GET -L localhost:9200/_cat/indices?v
-{% endif %}
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index e33d0cc41..64ff5ca6b 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -612,16 +612,6 @@ if [ $is_airgap -eq 0 ]; then
else
update_registry
update_docker_containers "soup"
- FEATURESCHECK=$(lookup_pillar features elastic)
- if [[ "$FEATURESCHECK" == "True" ]]; then
- TRUSTED_CONTAINERS=(
- "so-elasticsearch"
- "so-filebeat"
- "so-kibana"
- "so-logstash"
- )
- update_docker_containers "features" "-features"
- fi
fi
echo ""
echo "Stopping Salt Minion service."
diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml
index 016a123f0..7d86ccc04 100644
--- a/salt/curator/files/curator.yml
+++ b/salt/curator/files/curator.yml
@@ -12,11 +12,11 @@ client:
- {{elasticsearch}}
port: 9200
url_prefix:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %} use_ssl: True{% else %} use_ssl: False{% endif %}
+ use_ssl: True
certificate:
client_cert:
client_key:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %} ssl_no_validate: True{% else %} ssl_no_validate: False{% endif %}
+ ssl_no_validate: True
http_auth:
timeout: 30
master_only: False
diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml
index a22d65b7b..ad675b8ee 100644
--- a/salt/elastalert/defaults.yaml
+++ b/salt/elastalert/defaults.yaml
@@ -16,8 +16,8 @@ elastalert:
#aws_region: us-east-1
#profile: test
#es_url_prefix: elasticsearch
- #use_ssl: True
- #verify_certs: True
+ use_ssl: true
+ verify_certs: false
#es_send_get_body_as: GET
#es_username: someusername
#es_password: somepassword
diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls
index 9008832f1..8fcb46cda 100644
--- a/salt/elastalert/init.sls
+++ b/salt/elastalert/init.sls
@@ -104,8 +104,9 @@ elastaconf:
wait_for_elasticsearch:
module.run:
- http.wait_for_successful_query:
- - url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*'
+ - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*'
- wait_for: 180
+ - verify_ssl: False
so-elastalert:
docker_container.running:
diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index 2e20a878b..fed45bf79 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -9,12 +9,6 @@
{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %}
cluster.name: "{{ ESCLUSTERNAME }}"
network.host: 0.0.0.0
-
-# minimum_master_nodes need to be explicitly set when bound on a public IP
-# set to 1 to allow single node clusters
-# Details: https://github.com/elastic/elasticsearch/pull/17288
-#discovery.zen.minimum_master_nodes: 1
-# This is a test -- if this is here, then the volume is mounted correctly.
path.logs: /var/log/elasticsearch
action.destructive_requires_name: true
transport.bind_host: 0.0.0.0
@@ -25,17 +19,12 @@ cluster.routing.allocation.disk.watermark.low: 95%
cluster.routing.allocation.disk.watermark.high: 98%
cluster.routing.allocation.disk.watermark.flood_stage: 98%
xpack.ml.enabled: false
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
xpack.security.enabled: true
-{%- else %}
-xpack.security.enabled: false
-{%- endif %}
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: none
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.client_authentication: none
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
@@ -46,7 +35,6 @@ xpack.security.authc:
username: anonymous_user
roles: superuser
authz_exception: true
-{%- endif %}
node.name: {{ grains.host }}
script.max_compilations_rate: 1000/1m
{%- if TRUECLUSTER is sameas true %}
diff --git a/salt/elasticsearch/files/ingest/osquery.live_query b/salt/elasticsearch/files/ingest/osquery.live_query
new file mode 100644
index 000000000..92f325e1d
--- /dev/null
+++ b/salt/elasticsearch/files/ingest/osquery.live_query
@@ -0,0 +1,16 @@
+{
+ "description" : "osquery live query",
+ "processors" : [
+ {
+ "script": {
+ "lang": "painless",
+ "source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; "
+ }
+ },
+ { "remove": { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } },
+ { "rename": { "field": "distributed_query_execution_id", "target_field": "result.query_id", "ignore_missing": true } },
+ { "rename": { "field": "computer_name", "target_field": "host.hostname", "ignore_missing": true } },
+ { "pipeline": { "name": "osquery.normalize" } },
+ { "pipeline": { "name": "common" } }
+ ]
+}
diff --git a/salt/elasticsearch/files/ingest/osquery.normalize b/salt/elasticsearch/files/ingest/osquery.normalize
new file mode 100644
index 000000000..ce0a6ca92
--- /dev/null
+++ b/salt/elasticsearch/files/ingest/osquery.normalize
@@ -0,0 +1,14 @@
+{
+ "description" : "osquery normalize",
+ "processors" : [
+ { "rename": { "field": "result.columns.cmdline", "target_field": "process.command_line", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.cwd", "target_field": "process.working_directory", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.name", "target_field": "process.name", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.path", "target_field": "process.executable", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.pid", "target_field": "process.pid", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.parent", "target_field": "process.ppid", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.uid", "target_field": "user.id", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.username", "target_field": "user.name", "ignore_missing": true } },
+ { "rename": { "field": "result.columns.gid", "target_field": "group.id", "ignore_missing": true } }
+ ]
+}
diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result
index b6b4f22ef..9bb381946 100644
--- a/salt/elasticsearch/files/ingest/osquery.query_result
+++ b/salt/elasticsearch/files/ingest/osquery.query_result
@@ -1,24 +1,19 @@
{
"description" : "osquery",
"processors" : [
- { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
- { "gsub": { "field": "message2.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } },
- { "rename": { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns", "target_field": "winlog", "ignore_missing": true } },
+ { "json": { "field": "message", "target_field": "result", "ignore_failure": true } },
+ { "gsub": { "field": "result.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } },
+ { "rename": { "if": "ctx.result.columns?.eventid != null", "field": "result.columns", "target_field": "winlog", "ignore_missing": true } },
{ "json": { "field": "winlog.data", "target_field": "unparsed", "ignore_failure": true} },
{ "set": { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true } },
{ "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } },
{ "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } },
{ "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } },
{ "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } },
- { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } },
- {
- "script": {
- "lang": "painless",
- "source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; "
- }
- },
+ { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } },
{ "set": { "field": "event.module", "value": "osquery", "override": false } },
- { "set": { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false} },
+ { "set": { "field": "event.dataset", "value": "{{result.name}}", "override": false} },
+ { "pipeline": { "if": "!(ctx.containsKey('winlog'))", "name": "osquery.normalize" } },
{ "pipeline": { "name": "common" } }
]
}
diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines
index dce6a081b..fca50b7d4 100755
--- a/salt/elasticsearch/files/so-elasticsearch-pipelines
+++ b/salt/elasticsearch/files/so-elasticsearch-pipelines
@@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
- {% if grains['role'] in ['so-node','so-heavynode'] %}
- curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% else %}
- curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
- {% endif %}
+ curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -51,11 +47,7 @@ fi
cd ${ELASTICSEARCH_INGEST_PIPELINES}
echo "Loading pipelines..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% else %}
-for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% endif %}
echo
cd - >/dev/null
diff --git a/salt/elasticsearch/files/sotls.yml b/salt/elasticsearch/files/sotls.yml
deleted file mode 100644
index 2fc577337..000000000
--- a/salt/elasticsearch/files/sotls.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-keystore.path: /usr/share/elasticsearch/config/sokeys
-keystore.password: changeit
-keystore.algorithm: SunX509
-truststore.path: /etc/pki/java/cacerts
-truststore.password: changeit
-truststore.algorithm: PKIX
-protocols:
-- TLSv1.2
-ciphers:
-- TLS_RSA_WITH_AES_128_CBC_SHA256
-- TLS_RSA_WITH_AES_256_GCM_SHA384
-transport.encrypted: true
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
-http.encrypted: true
-{%- else %}
-http.encrypted: false
-{%- endif %}
diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls
index debb37512..df297986a 100644
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -140,14 +140,6 @@ esyml:
- group: 939
- template: jinja
-sotls:
- file.managed:
- - name: /opt/so/conf/elasticsearch/sotls.yml
- - source: salt://elasticsearch/files/sotls.yml
- - user: 930
- - group: 939
- - template: jinja
-
#sync templates to /opt/so/conf/elasticsearch/templates
{% for TEMPLATE in TEMPLATES %}
es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
@@ -199,7 +191,7 @@ so-elasticsearch:
{% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
- discovery.type=single-node
{% endif %}
- - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }}
+ - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true
ulimits:
- memlock=-1:-1
- nofile=65536:65536
@@ -221,7 +213,6 @@ so-elasticsearch:
- /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
- /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
- - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro
- watch:
- file: cacertz
- file: esyml
diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json
index 062838670..012c590d0 100644
--- a/salt/elasticsearch/templates/so/so-common-template.json
+++ b/salt/elasticsearch/templates/so/so-common-template.json
@@ -365,6 +365,10 @@
"request":{
"type":"object",
"dynamic": true
+ },
+ "result":{
+ "type":"object",
+ "dynamic": true
},
"rfb":{
"type":"object",
diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 1b902d542..b6aa218ef 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -260,7 +260,7 @@ output.{{ type }}:
{%- if grains['role'] in ["so-eval", "so-import"] %}
output.elasticsearch:
enabled: true
- hosts: ["{{ MANAGER }}:9200"]
+ hosts: ["https://{{ MANAGER }}:9200"]
pipelines:
- pipeline: "%{[module]}.%{[dataset]}"
indices:
diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index 91c6cdbb8..339d307ee 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -13,7 +13,6 @@
# along with this program. If not, see .
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls in allowed_states %}
-
{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{% set LOCALHOSTNAME = salt['grains.get']('host') %}
diff --git a/salt/kibana/bin/keepkibanahappy.sh b/salt/kibana/bin/keepkibanahappy.sh
deleted file mode 100644
index 541a666bd..000000000
--- a/salt/kibana/bin/keepkibanahappy.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-# Wait for ElasticSearch to come up, so that we can query for version infromation
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 30 ]]; do
- curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
- if [ $? -eq 0 ]; then
- ELASTICSEARCH_CONNECTED="yes"
- echo "connected!"
- break
- else
- ((COUNT+=1))
- sleep 1
- echo -n "."
- fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
- echo
- echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'"
- echo
-
- exit
-fi
-
-# Make sure Kibana is running
-MAX_WAIT=240
-
-# Check to see if Kibana is available
-wait_step=0
- until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do
- wait_step=$(( ${wait_step} + 1 ))
- echo "Waiting on Kibana...Attempt #$wait_step"
- if [ ${wait_step} -gt ${MAX_WAIT} ]; then
- echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
- exit 5
- fi
- sleep 1s;
- done
-
-
-# Apply Kibana template
- echo
- echo "Applying Kibana template..."
- curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \
- -H 'Content-Type: application/json' \
- -d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}'
- echo
-
- curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \
- -H 'Content-Type: application/json' \
- -d'{"index" : {"number_of_replicas" : 0}}'
- echo
diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load
index e545dc452..b40227031 100644
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -14,5 +14,9 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o
# SOCtopus and Manager
sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson
+## This hackery will be removed if using Elastic Auth ##
+
+# Let's snag a cookie from Kibana
+THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
# Load saved objects
-curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
+curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml
index 4b9424b4c..856f87909 100644
--- a/salt/kibana/etc/kibana.yml
+++ b/salt/kibana/etc/kibana.yml
@@ -1,11 +1,11 @@
---
# Default Kibana configuration from kibana-docker.
{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
server.name: kibana
server.host: "0"
server.basePath: /kibana
-elasticsearch.hosts: [ "http://{{ ES }}:9200" ]
+elasticsearch.hosts: [ "https://{{ ES }}:9200" ]
+elasticsearch.ssl.verificationMode: none
#kibana.index: ".kibana"
#elasticsearch.username: elastic
#elasticsearch.password: changeme
@@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000
logging.dest: /var/log/kibana/kibana.log
telemetry.enabled: false
security.showInsecureClusterWarning: false
+xpack.security.authc.providers:
+ anonymous.anonymous1:
+ order: 0
+ credentials: "elasticsearch_anonymous_user"
diff --git a/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja
new file mode 100644
index 000000000..83aa0c02d
--- /dev/null
+++ b/salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja
@@ -0,0 +1,19 @@
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
+{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
+
+input {
+ redis {
+ host => '{{ MANAGER }}'
+ port => 6379
+ data_type => 'pattern_channel'
+ key => 'results_*'
+ type => 'live_query'
+ add_field => {
+ "module" => "osquery"
+ "dataset" => "live_query"
+ }
+ threads => {{ THREADS }}
+ batch_count => {{ BATCH }}
+ }
+}
diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
index d1764eecc..d17dc2b22 100644
--- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "zeek" and "import" not in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-zeek"
template => "/templates/so-zeek-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
index 35900471e..4562dcee7 100644
--- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if "import" in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-import"
template => "/templates/so-import-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
index 0a7d961de..fb6eaee5d 100644
--- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [event_type] == "sflow" {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-flow"
template => "/templates/so-flow-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
index 7f0e30fbc..61aa21a82 100644
--- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [event_type] == "ids" and "import" not in [tags] {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-ids"
template => "/templates/so-ids-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
index 4a27428f7..0afbf45ea 100644
--- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "syslog" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-syslog"
template => "/templates/so-syslog-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
index 2a71e3fab..efa46c7af 100644
--- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
@@ -3,9 +3,8 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
- if [module] =~ "osquery" {
+ if [module] =~ "osquery" and "live_query" not in [dataset] {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
@@ -13,10 +12,8 @@ output {
template_name => "so-osquery"
template => "/templates/so-osquery-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja
new file mode 100644
index 000000000..51e691176
--- /dev/null
+++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja
@@ -0,0 +1,43 @@
+{%- if grains['role'] == 'so-eval' -%}
+{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
+{%- else %}
+{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{%- endif %}
+{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
+
+filter {
+ if [type] =~ "live_query" {
+
+ mutate {
+ rename => {
+ "[host][hostname]" => "computer_name"
+ }
+ }
+
+ prune {
+ blacklist_names => ["host"]
+ }
+
+ split {
+ field => "rows"
+ }
+ }
+}
+
+
+output {
+ if [type] =~ "live_query" {
+ elasticsearch {
+ pipeline => "osquery.live_query"
+ hosts => "{{ ES }}"
+ index => "so-osquery"
+ template_name => "so-osquery"
+ template => "/templates/so-osquery-template.json"
+ template_overwrite => true
+ {%- if grains['role'] in ['so-node','so-heavynode'] %}
+ ssl => true
+ ssl_certificate_verification => false
+ {%- endif %}
+ }
+ }
+}
diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
index 9b93d327b..764f597b9 100644
--- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [dataset] =~ "firewall" {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-firewall"
template => "/templates/so-firewall-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
index 29837040a..5013bafc1 100644
--- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "suricata" and "import" not in [tags] {
elasticsearch {
@@ -12,10 +11,8 @@ output {
index => "so-ids"
template_name => "so-ids"
template => "/templates/so-ids-template.json"
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
index beaf24727..349c0ada1 100644
--- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if "beat-ext" in [tags] and "import" not in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-beats"
template => "/templates/so-beats-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
index 95c81577b..1a4987a53 100644
--- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "ossec" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-ossec"
template => "/templates/so-ossec-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
index 1e8c44cc6..d564486e4 100644
--- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "strelka" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-strelka"
template => "/templates/so-strelka-template.json"
template_overwrite => true
- {%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
- {%- endif %}
}
}
}
diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json
index b8dc5eb21..5f3a359b5 100644
--- a/salt/soc/files/soc/hunt.queries.json
+++ b/salt/soc/files/soc/hunt.queries.json
@@ -42,6 +42,7 @@
{ "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.dataset:mysql | groupby mysql.command"},
{ "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.dataset:notice | groupby notice.note notice.message"},
{ "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.dataset:ntlm | groupby ntlm.server.dns.name"},
+ { "name": "Osquery Live Queries", "description": "Osquery Live Query results grouped by computer name", "query": "event.dataset:live_query | groupby host.hostname"},
{ "name": "PE", "description": "PE files list", "query": "event.dataset:pe | groupby file.machine file.os file.subsystem"},
{ "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.dataset:radius | groupby user.name.keyword"},
{ "name": "RDP", "description": "RDP grouped by client name", "query": "event.dataset:rdp | groupby client.name"},
diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index bda2df431..520f87b93 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -1,7 +1,6 @@
{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
{%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %}
{%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
{%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
{%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %}
{%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %}
@@ -31,7 +30,7 @@
"hostUrl": "http://{{ MANAGERIP }}:4434/"
},
"elastic": {
- "hostUrl": "http://{{ MANAGERIP }}:9200",
+ "hostUrl": "https://{{ MANAGERIP }}:9200",
{%- if salt['pillar.get']('nodestab', {}) %}
"remoteHostUrls": [
{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
@@ -45,7 +44,7 @@
},
"sostatus": {
"refreshIntervalMs": 30000,
- "offlineThresholdMs": 60000
+ "offlineThresholdMs": 900000
},
{% if THEHIVEKEY != '' %}
"thehive": {
diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf
index 4f58ecf83..c9941c3e1 100644
--- a/salt/soctopus/files/SOCtopus.conf
+++ b/salt/soctopus/files/SOCtopus.conf
@@ -6,7 +6,7 @@
[es]
-es_url = http://{{MANAGER}}:9200
+es_url = https://{{MANAGER}}:9200
es_ip = {{MANAGER}}
es_user = YOURESUSER
es_pass = YOURESPASS
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index 0c447172f..b8976b8c9 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -622,7 +622,7 @@
# ## specify a list of one or more Elasticsearch servers
# # you can add username and password to your url to use basic authentication:
# # servers = ["http://user:pass@localhost:9200"]
- servers = ["http://{{ MANAGER }}:9200"]
+ servers = ["https://{{ MANAGER }}:9200"]
{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]
diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams
index 490c7b548..3838f67df 100644
--- a/salt/utility/bin/crossthestreams
+++ b/salt/utility/bin/crossthestreams
@@ -1,7 +1,6 @@
#!/bin/bash
{% set ES = salt['pillar.get']('manager:mainip', '') %}
{% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
# Wait for ElasticSearch to come up, so that we can query for version infromation
@@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
- curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
+ curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
fi
echo "Applying cross cluster search config..."
- curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+ curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
-H 'Content-Type: application/json' \
-d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
@@ -37,7 +36,7 @@ echo "Applying cross cluster search config..."
{%- if TRUECLUSTER is sameas false %}
{%- if salt['pillar.get']('nodestab', {}) %}
{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-curl -XPUT -L http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
+curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
{%- endfor %}
{%- endif %}
{%- endif %}
diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval
index f63a61942..dcf46de7a 100644
--- a/salt/utility/bin/eval
+++ b/salt/utility/bin/eval
@@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
- curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
+ curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
fi
echo "Applying cross cluster search config..."
- curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
+ curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
-H 'Content-Type: application/json' \
-d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"