diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 217d69036..b82890faa 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -24,7 +24,7 @@ mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs -{% if grains.role == 'so-import' %} +{% if grains.role in ['so-import', 'so-standalone', 'so-eval'] %} # Add SO-Manager Elasticsearch Ouput ESCACRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) JSON_STRING=$( jq -n \ diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status b/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status new file mode 100644 index 000000000..db31dcb0f --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status @@ -0,0 +1,15 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +if [ "$1" == "" ]; then + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_all/_ilm/explain | jq . +else + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/$1/_ilm/explain | jq .[] +fi diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete new file mode 100644 index 000000000..108dd1178 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete @@ -0,0 +1,11 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X DELETE https://{{ NODEIP }}:9200/_ilm/policy/$1 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load new file mode 100644 index 000000000..09cc6dc08 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load @@ -0,0 +1,73 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +# Set up ILM policies +echo +echo "Setting up default Security Onion index lifecycle management policies..." + +# Elasticsearch logs +echo +echo "Setting up Elasticsearch ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/elasticsearch-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Import logs +echo +echo "Setting up Import ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-import-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Kibana logs +echo +echo "Setting up Kibana ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kibana-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Kratos logs +echo +echo "Setting up Kratos ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kratos-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Logstash logs +echo +echo "Setting up Logstash ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-logstash-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Redis logs +echo +echo "Setting up Redis ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-redis-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Strelka logs +echo +echo "Setting up Strelka ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-strelka-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Suricata logs +echo +echo "Setting up Suricata ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-suricata-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Syslog logs +echo +echo "Setting up Syslog ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-syslog-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo + +# Zeek logs +echo +echo "Setting up Zeek ILM policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-zeek-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' +echo diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view new file mode 100644 index 000000000..d69e328fe --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view @@ -0,0 +1,15 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +if [ "$1" == "" ]; then + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy | jq . +else + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy/$1 | jq .[] +fi diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-restart b/salt/common/tools/sbin/so-elasticsearch-ilm-restart new file mode 100644 index 000000000..7f422ed6e --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-restart @@ -0,0 +1,10 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +so-elasticsearch-ilm-stop +so-elasticsearch-ilm-start diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-start b/salt/common/tools/sbin/so-elasticsearch-ilm-start new file mode 100644 index 000000000..98dd38e9e --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-start @@ -0,0 +1,12 @@ +/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +echo "Starting ILM..." +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/start diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-status b/salt/common/tools/sbin/so-elasticsearch-ilm-status new file mode 100644 index 000000000..8d78adc5b --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-status @@ -0,0 +1,11 @@ +/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/status | jq . diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-stop b/salt/common/tools/sbin/so-elasticsearch-ilm-stop new file mode 100644 index 000000000..23c068918 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-stop @@ -0,0 +1,12 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +echo "Stopping ILM..." +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/stop diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 5f0a7ea79..553144b45 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,7 +1,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% set REDIS_NODES = [] %} {% set LOGSTASH_NODES = [] %} -{% set node_data = salt['pillar.get']('logstash:nodes') %} +{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} {% for node_type, node_details in node_data.items() | sort %} {% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %}