CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update verbiage for so-curator-cluster-delete-delete

Browse Source
This commit is contained in:
Wes
2023-03-28 14:18:27 +00:00
parent 138b312705
commit e1bda5acfd
1 changed files with 26 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
salt/curator/files/bin/so-curator-cluster-delete-delete
View File

@@ -9,11 +9,6 @@
{%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%} {%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%}
{%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%} {%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%}
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
LOG="/opt/so/log/curator/so-curator-cluster-delete.log" LOG="/opt/so/log/curator/so-curator-cluster-delete.log"
LOG_SIZE_LIMIT=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}}) LOG_SIZE_LIMIT=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}})
@@ -26,22 +21,26 @@ closedindices() {
# If we can't query Elasticsearch, then immediately return false. # If we can't query Elasticsearch, then immediately return false.
/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close > /dev/null 2>&1 /usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close > /dev/null 2>&1
[ $? -eq 1 ] && return false [ $? -eq 1 ] && return false
# First, get the list of closed indices using _cat/indices?h=index,status | grep close | awk '{print $1}'. # We need to determine the oldest closed index.
# First, get the list of closed indices using _cat/indices?h=index,status | grep close | awk '{print $1}'.
# Next, filter out any so-case indices. # Next, filter out any so-case indices.
# Finally, use grep's -q option to return true if there are any remaining logstash-, so-, or .ds-logs- indices. # Finally, use grep's -q option to return true if there are any remaining logstash-, so-, or .ds-logs- indices.
CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close | awk '{print $1}' | grep -v "so-case" | grep -q -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close | awk '{print $1}' | grep -v "so-case" | grep -q -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
for CLOSED_INDEX in ${CLOSED_INDICES}; do # We iterate through the closed indices
# Now that we've determined OLDEST_OPEN_INDEX, ask Elasticsearch to delete it. for CLOSED_INDEX in ${CLOSED_INDICES}; do
# First, we need to check if the index is assigned as the current write index for a data stream # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
# To do so, we need to identify to which data stream this index is associated # To do so, we need to identify to which data stream this index is associated
# We extract the data stream name using the pattern below
DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
DATASTREAM=$(echo "${CLOSED_INDEX}" | grep -oE "$DATASTREAM_PATTERN") DATASTREAM=$(echo "${CLOSED_INDEX}" | grep -oE "$DATASTREAM_PATTERN")
# We look up the data stream, and determine the write index
CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
if [ "${CLOSED_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then # We make sure we are not trying to delete a write index
# This should not be a write index, so we should be allowed to delete it if [ "${CLOSED_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
/usr/sbin/so-elasticsearch-query ${CLOSED_INDEX} -XDELETE # This should not be a write index, so we should be allowed to delete it
# Finally, write a log entry that says we deleted it. /usr/sbin/so-elasticsearch-query ${CLOSED_INDEX} -XDELETE
echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${CLOSED_INDEX} deleted ..." >> ${LOG} # Finally, write a log entry that says we deleted it.
echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${CLOSED_INDEX} deleted ..." >> ${LOG}
fi fi
if ! overlimit; then if ! overlimit; then
exit exit
@@ -51,25 +50,26 @@ closedindices() {
while overlimit; do while overlimit; do
# We need to determine OLDEST_OPEN_INDEX: # We need to determine the oldest open index.
# First, get the list of open indices using _cat/indices?h=index,status | grep open | awk '{print $1}'. # First, get the list of open indices using _cat/indices?h=index,status | grep open | awk '{print $1}'.
# Next, filter out any so-case indices and only select the remaining logstash-, so-, or .ds-logs- indices. # Next, filter out any so-case indices and only select the remaining logstash-, so-, or .ds-logs- indices.
# Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field. # Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field.
OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep open | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep open | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
#OLDEST_OPEN_INDEX=$(so-elasticsearch-query _cat/indices?h=index,status | grep open | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3 | head -1) # We iterate through the open indices
for OPEN_INDEX in ${OPEN_INDICES}; do
for OPEN_INDEX in ${OPEN_INDICES}; do # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
# Now that we've determined OLDEST_OPEN_INDEX, ask Elasticsearch to delete it.
# First, we need to check if the index is assigned as the current write index for a data stream
# To do so, we need to identify to which data stream this index is associated # To do so, we need to identify to which data stream this index is associated
# We extract the data stream name using the pattern below
DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
DATASTREAM=$(echo "${OPEN_INDEX}" | grep -oE "$DATASTREAM_PATTERN") DATASTREAM=$(echo "${OPEN_INDEX}" | grep -oE "$DATASTREAM_PATTERN")
CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) # We look up the data stream, and determine the write index
if [ "${OPEN_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
# This should not be a write index, so we should be allowed to delete it # We make sure we are not trying to delete a write index
/usr/sbin/so-elasticsearch-query ${OPEN_INDEX} -XDELETE if [ "${OPEN_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
# Finally, write a log entry that says we deleted it. # This should not be a write index, so we should be allowed to delete it
echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${OPEN_INDEX} deleted ..." >> ${LOG} /usr/sbin/so-elasticsearch-query ${OPEN_INDEX} -XDELETE
# Finally, write a log entry that says we deleted it.
echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${OPEN_INDEX} deleted ..." >> ${LOG}
fi fi
if ! overlimit; then if ! overlimit; then
exit exit