diff --git a/salt/curator/files/bin/so-curator-cluster-delete-delete b/salt/curator/files/bin/so-curator-cluster-delete-delete index 4f136274e..7d67a76ff 100755 --- a/salt/curator/files/bin/so-curator-cluster-delete-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete-delete @@ -38,18 +38,21 @@ closedindices() { while overlimit && closedindices; do CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) - for CLOSED_INDEX in ${CLOSED_INDICES}; do - # Now that we've determined OLDEST_OPEN_INDEX, ask Elasticsearch to delete it. - # First, we need to check if the index is assigned as the current write index for a data stream + # We iterate through the closed indices + for CLOSED_INDEX in ${CLOSED_INDICES}; do + # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream # To do so, we need to identify to which data stream this index is associated + # We extract the data stream name using the pattern below DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" DATASTREAM=$(echo "${CLOSED_INDEX}" | grep -oE "$DATASTREAM_PATTERN") + # We look up the data stream, and determine the write index CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) + # We make sure we are not trying to delete a write index if [ "${CLOSED_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then - # This should not be a write index, so we should be allowed to delete it - /usr/sbin/so-elasticsearch-query ${CLOSED_INDEX} -XDELETE - # Finally, write a log entry that says we deleted it. - echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${CLOSED_INDEX} deleted ..." >> ${LOG} + # This should not be a write index, so we should be allowed to delete it + /usr/sbin/so-elasticsearch-query ${CLOSED_INDEX} -XDELETE + # Finally, write a log entry that says we deleted it. + echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${CLOSED_INDEX} deleted ..." >> ${LOG} fi if ! overlimit; then exit @@ -59,25 +62,26 @@ done while overlimit; do - # We need to determine OLDEST_OPEN_INDEX: + # We need to determine the oldest open index. # First, get the list of open indices using _cat/indices?h=index,status | grep open | awk '{print $1}'. - # Next, filter out any so-case indices and only select the remaining logstash-, so-, or .ds-logs- indices. - # Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field. + # Next, filter out any so-case indices and only select the remaining logstash-, so-, or .ds-logs- indices. + # Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field. OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep open | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) - #OLDEST_OPEN_INDEX=$(so-elasticsearch-query _cat/indices?h=index,status | grep open | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3 | head -1) - + # We iterate through the open indices for OPEN_INDEX in ${OPEN_INDICES}; do - # Now that we've determined OLDEST_OPEN_INDEX, ask Elasticsearch to delete it. - # First, we need to check if the index is assigned as the current write index for a data stream + # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream # To do so, we need to identify to which data stream this index is associated + # We extract the data stream name using the pattern below DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" DATASTREAM=$(echo "${OPEN_INDEX}" | grep -oE "$DATASTREAM_PATTERN") + # We look up the data stream, and determine the write index CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) + # We make sure we are not trying to delete a write index if [ "${OPEN_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then - # This should not be a write index, so we should be allowed to delete it - /usr/sbin/so-elasticsearch-query ${OPEN_INDEX} -XDELETE - # Finally, write a log entry that says we deleted it. - echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${OPEN_INDEX} deleted ..." >> ${LOG} + # This should not be a write index, so we should be allowed to delete it + /usr/sbin/so-elasticsearch-query ${OPEN_INDEX} -XDELETE + # Finally, write a log entry that says we deleted it. + echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${OPEN_INDEX} deleted ..." >> ${LOG} fi if ! overlimit; then exit