From a945768251b434a14e90dc8010d6f51eefad1e47 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 11 Dec 2025 11:15:30 -0500 Subject: [PATCH] Refactor backup --- salt/elasticsearch/defaults.yaml | 13 ++++ salt/manager/tools/sbin/soup | 56 ++++++++--------- salt/soc/files/soc/so-detections-backup.py | 72 +++++++++++++--------- 3 files changed, 82 insertions(+), 59 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 5cfb9a0e0..c9f77aa7d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -299,6 +299,19 @@ elasticsearch: hot: actions: {} min_age: 0ms + sos-backup: + index_sorting: false + index_template: + composed_of: [] + ignore_missing_component_templates: [] + index_patterns: + - sos-backup-* + priority: 501 + template: + settings: + index: + number_of_replicas: 0 + number_of_shards: 1 so-assistant-chat: index_sorting: false index_template: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 9fd9542c0..5029f28c3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1125,40 +1125,35 @@ mkdir -p /nsm/backup/detections-migration/2-4-200 cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200 cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200 -if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then - python3 /opt/so/conf/soc/so-detections-backup.py +# Backup so-detection index via reindex +echo "Creating sos-backup index template..." +template_result=$(/sbin/so-elasticsearch-query '_index_template/sos-backup' -X PUT \ + --retry 5 --retry-delay 15 --retry-all-errors \ + -d '{"index_patterns":["sos-backup-*"],"priority":501,"template":{"settings":{"index":{"number_of_replicas":0,"number_of_shards":1}}}}') - # Verify backup by comparing counts - echo "Verifying detection overrides backup..." - es_override_count=$(/sbin/so-elasticsearch-query 'so-detection/_count' \ - --retry 5 --retry-delay 10 --retry-all-errors \ - -d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') || { - echo " Error: Failed to query Elasticsearch for override count" - exit 1 - } +if [[ -z "$template_result" ]] || ! echo "$template_result" | jq -e '.acknowledged == true' > /dev/null 2>&1; then + echo "Error: Failed to create sos-backup index template" + echo "$template_result" + exit 1 +fi - if [[ ! "$es_override_count" =~ ^[0-9]+$ ]]; then - echo " Error: Invalid override count from Elasticsearch: '$es_override_count'" - exit 1 - fi +BACKUP_INDEX="sos-backup-detection-$(date +%Y%m%d-%H%M%S)" +echo "Backing up so-detection index to $BACKUP_INDEX..." +reindex_result=$(/sbin/so-elasticsearch-query '_reindex?wait_for_completion=true' \ + --retry 5 --retry-delay 15 --retry-all-errors \ + -X POST -d "{\"source\": {\"index\": \"so-detection\"}, \"dest\": {\"index\": \"$BACKUP_INDEX\"}}") - backup_override_count=$(find /nsm/backup/detections/repo/*/overrides -type f 2>/dev/null | wc -l) - - echo " Elasticsearch overrides: $es_override_count" - echo " Backed up overrides: $backup_override_count" - - if [[ "$es_override_count" -gt 0 ]]; then - if [[ "$backup_override_count" -gt 0 ]]; then - echo " Override backup verified successfully" - else - echo " Error: Elasticsearch has $es_override_count overrides but backup has 0 files" - exit 1 - fi - else - echo " No overrides to backup" - fi +if [[ -z "$reindex_result" ]]; then + echo "Error: Backup of detections failed - no response from Elasticsearch" + exit 1 +elif echo "$reindex_result" | jq -e '.created >= 0' > /dev/null 2>&1; then + echo "Backup complete: $(echo "$reindex_result" | jq -r '.created') documents copied" +elif echo "$reindex_result" | grep -q "index_not_found_exception"; then + echo "so-detection index does not exist, skipping backup" else - echo "SOC Detections backup script not found, skipping detection backup" + echo "Error: Backup of detections failed" + echo "$reindex_result" + exit 1 fi } @@ -1304,6 +1299,7 @@ fi echo "Removing idstools symlink and scripts..." rm -rf /usr/sbin/so-idstools* sed -i '/^#\?so-idstools$/d' /opt/so/conf/so-status/so-status.conf +crontab -l | grep -v 'so-rule-update' | crontab - # Backup the salt master config & manager pillar before editing it cp /opt/so/saltstack/local/pillar/minions/$MINIONID.sls /nsm/backup/detections-migration/2-4-200/ diff --git a/salt/soc/files/soc/so-detections-backup.py b/salt/soc/files/soc/so-detections-backup.py index 085b1e4c7..0300c15f2 100644 --- a/salt/soc/files/soc/so-detections-backup.py +++ b/salt/soc/files/soc/so-detections-backup.py @@ -6,6 +6,7 @@ # This script queries Elasticsearch for Custom Detections and all Overrides, # and git commits them to disk at $OUTPUT_DIR +import argparse import os import subprocess import json @@ -18,10 +19,10 @@ from datetime import datetime urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Constants -ES_URL = "https://localhost:9200/so-detection/_search" +DEFAULT_INDEX = "so-detection" +DEFAULT_OUTPUT_DIR = "/nsm/backup/detections/repo" QUERY_DETECTIONS = '{"query": {"bool": {"must": [{"match_all": {}}, {"term": {"so_detection.ruleset": "__custom__"}}]}},"size": 10000}' QUERY_OVERRIDES = '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}},"size": 10000}' -OUTPUT_DIR = "/nsm/backup/detections/repo" AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config" def get_auth_credentials(auth_file): @@ -30,9 +31,10 @@ def get_auth_credentials(auth_file): if line.startswith('user ='): return line.split('=', 1)[1].strip().replace('"', '') -def query_elasticsearch(query, auth): +def query_elasticsearch(query, auth, index): + url = f"https://localhost:9200/{index}/_search" headers = {"Content-Type": "application/json"} - response = requests.get(ES_URL, headers=headers, data=query, auth=auth, verify=False) + response = requests.get(url, headers=headers, data=query, auth=auth, verify=False) response.raise_for_status() return response.json() @@ -47,12 +49,12 @@ def save_content(hit, base_folder, subfolder="", extension="txt"): f.write(content) return file_path -def save_overrides(hit): +def save_overrides(hit, output_dir): so_detection = hit["_source"]["so_detection"] public_id = so_detection["publicId"] overrides = so_detection["overrides"] language = so_detection["language"] - folder = os.path.join(OUTPUT_DIR, language, "overrides") + folder = os.path.join(output_dir, language, "overrides") os.makedirs(folder, exist_ok=True) extension = "yaml" if language == "sigma" else "txt" file_path = os.path.join(folder, f"{public_id}.{extension}") @@ -60,20 +62,20 @@ def save_overrides(hit): f.write('\n'.join(json.dumps(override) for override in overrides) if isinstance(overrides, list) else overrides) return file_path -def ensure_git_repo(): - if not os.path.isdir(os.path.join(OUTPUT_DIR, '.git')): +def ensure_git_repo(output_dir): + if not os.path.isdir(os.path.join(output_dir, '.git')): subprocess.run(["git", "config", "--global", "init.defaultBranch", "main"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "init"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "remote", "add", "origin", "default"], check=True) + subprocess.run(["git", "-C", output_dir, "init"], check=True) + subprocess.run(["git", "-C", output_dir, "remote", "add", "origin", "default"], check=True) -def commit_changes(): - ensure_git_repo() - subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.email", "securityonion@local.invalid"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.name", "securityonion"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "add", "."], check=True) - status_result = subprocess.run(["git", "-C", OUTPUT_DIR, "status"], capture_output=True, text=True) +def commit_changes(output_dir): + ensure_git_repo(output_dir) + subprocess.run(["git", "-C", output_dir, "config", "user.email", "securityonion@local.invalid"], check=True) + subprocess.run(["git", "-C", output_dir, "config", "user.name", "securityonion"], check=True) + subprocess.run(["git", "-C", output_dir, "add", "."], check=True) + status_result = subprocess.run(["git", "-C", output_dir, "status"], capture_output=True, text=True) print(status_result.stdout) - commit_result = subprocess.run(["git", "-C", OUTPUT_DIR, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True) + commit_result = subprocess.run(["git", "-C", output_dir, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True) if commit_result.returncode == 1: print("No changes to commit.") elif commit_result.returncode == 0: @@ -81,29 +83,41 @@ def commit_changes(): else: commit_result.check_returncode() +def parse_args(): + parser = argparse.ArgumentParser(description="Backup custom detections and overrides from Elasticsearch") + parser.add_argument("--output", "-o", default=DEFAULT_OUTPUT_DIR, + help=f"Output directory for backups (default: {DEFAULT_OUTPUT_DIR})") + parser.add_argument("--index", "-i", default=DEFAULT_INDEX, + help=f"Elasticsearch index to query (default: {DEFAULT_INDEX})") + return parser.parse_args() + def main(): + args = parse_args() + output_dir = args.output + index = args.index + try: timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - print(f"Backing up Custom Detections and all Overrides to {OUTPUT_DIR} - {timestamp}\n") - - os.makedirs(OUTPUT_DIR, exist_ok=True) + print(f"Backing up Custom Detections and all Overrides to {output_dir} - {timestamp}\n") + + os.makedirs(output_dir, exist_ok=True) auth_credentials = get_auth_credentials(AUTH_FILE) username, password = auth_credentials.split(':', 1) auth = HTTPBasicAuth(username, password) - + # Query and save custom detections - detections = query_elasticsearch(QUERY_DETECTIONS, auth)["hits"]["hits"] + detections = query_elasticsearch(QUERY_DETECTIONS, auth, index)["hits"]["hits"] for hit in detections: - save_content(hit, OUTPUT_DIR, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt") - + save_content(hit, output_dir, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt") + # Query and save overrides - overrides = query_elasticsearch(QUERY_OVERRIDES, auth)["hits"]["hits"] + overrides = query_elasticsearch(QUERY_OVERRIDES, auth, index)["hits"]["hits"] for hit in overrides: - save_overrides(hit) - - commit_changes() - + save_overrides(hit, output_dir) + + commit_changes(output_dir) + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") print(f"Backup Completed - {timestamp}") except Exception as e: