CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-02-01 12:53:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15301 from Security-Onion-Solutions/idstools-refactor

Browse Source 
Rework backup
This commit is contained in:
Josh Brower
2025-12-11 13:57:25 -05:00
committed by GitHub
parent c6646e3821 cb9a6fac25
commit 6fe817ca4a
4 changed files with 113 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
salt/elasticsearch/defaults.yaml
View File

@@ -299,6 +299,19 @@ elasticsearch:
hot:
actions: {}
min_age: 0ms
sos-backup:
index_sorting: false
index_template:
composed_of: []
ignore_missing_component_templates: []
index_patterns:
- sos-backup-*
priority: 501
template:
settings:
index:
number_of_replicas: 0
number_of_shards: 1
so-assistant-chat:
index_sorting: false
index_template:

56
salt/manager/tools/sbin/soup
View File

@@ -1125,40 +1125,35 @@ mkdir -p /nsm/backup/detections-migration/2-4-200
cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200
cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200
if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then
python3 /opt/so/conf/soc/so-detections-backup.py
# Backup so-detection index via reindex
echo "Creating sos-backup index template..."
template_result=$(/sbin/so-elasticsearch-query '_index_template/sos-backup' -X PUT \
--retry 5 --retry-delay 15 --retry-all-errors \
-d '{"index_patterns":["sos-backup-*"],"priority":501,"template":{"settings":{"index":{"number_of_replicas":0,"number_of_shards":1}}}}')
# Verify backup by comparing counts
echo "Verifying detection overrides backup..."
es_override_count=$(/sbin/so-elasticsearch-query 'so-detection/_count' \
--retry 5 --retry-delay 10 --retry-all-errors \
-d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') || {
echo " Error: Failed to query Elasticsearch for override count"
exit 1
}
if [[ -z "$template_result" ]] || ! echo "$template_result" | jq -e '.acknowledged == true' > /dev/null 2>&1; then
echo "Error: Failed to create sos-backup index template"
echo "$template_result"
exit 1
fi
if [[ ! "$es_override_count" =~ ^[0-9]+$ ]]; then
echo " Error: Invalid override count from Elasticsearch: '$es_override_count'"
exit 1
fi
BACKUP_INDEX="sos-backup-detection-$(date +%Y%m%d-%H%M%S)"
echo "Backing up so-detection index to $BACKUP_INDEX..."
reindex_result=$(/sbin/so-elasticsearch-query '_reindex?wait_for_completion=true' \
--retry 5 --retry-delay 15 --retry-all-errors \
-X POST -d "{\"source\": {\"index\": \"so-detection\"}, \"dest\": {\"index\": \"$BACKUP_INDEX\"}}")
backup_override_count=$(find /nsm/backup/detections/repo/*/overrides -type f 2>/dev/null | wc -l)
echo " Elasticsearch overrides: $es_override_count"
echo " Backed up overrides: $backup_override_count"
if [[ "$es_override_count" -gt 0 ]]; then
if [[ "$backup_override_count" -gt 0 ]]; then
echo " Override backup verified successfully"
else
echo " Error: Elasticsearch has $es_override_count overrides but backup has 0 files"
exit 1
fi
else
echo " No overrides to backup"
fi
if [[ -z "$reindex_result" ]]; then
echo "Error: Backup of detections failed - no response from Elasticsearch"
exit 1
elif echo "$reindex_result" | jq -e '.created >= 0' > /dev/null 2>&1; then
echo "Backup complete: $(echo "$reindex_result" | jq -r '.created') documents copied"
elif echo "$reindex_result" | grep -q "index_not_found_exception"; then
echo "so-detection index does not exist, skipping backup"
else
echo "SOC Detections backup script not found, skipping detection backup"
echo "Error: Backup of detections failed"
echo "$reindex_result"
exit 1
fi
}
@@ -1304,6 +1299,7 @@ fi
echo "Removing idstools symlink and scripts..."
rm -rf /usr/sbin/so-idstools*
sed -i '/^#\?so-idstools$/d' /opt/so/conf/so-status/so-status.conf
crontab -l | grep -v 'so-rule-update' | crontab -
# Backup the salt master config & manager pillar before editing it
cp /opt/so/saltstack/local/pillar/minions/$MINIONID.sls /nsm/backup/detections-migration/2-4-200/

72
salt/soc/files/soc/so-detections-backup.py
View File

@@ -6,6 +6,7 @@
# This script queries Elasticsearch for Custom Detections and all Overrides,
# and git commits them to disk at $OUTPUT_DIR
import argparse
import os
import subprocess
import json
@@ -18,10 +19,10 @@ from datetime import datetime
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Constants
ES_URL = "https://localhost:9200/so-detection/_search"
DEFAULT_INDEX = "so-detection"
DEFAULT_OUTPUT_DIR = "/nsm/backup/detections/repo"
QUERY_DETECTIONS = '{"query": {"bool": {"must": [{"match_all": {}}, {"term": {"so_detection.ruleset": "__custom__"}}]}},"size": 10000}'
QUERY_OVERRIDES = '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}},"size": 10000}'
OUTPUT_DIR = "/nsm/backup/detections/repo"
AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config"
def get_auth_credentials(auth_file):
@@ -30,9 +31,10 @@ def get_auth_credentials(auth_file):
if line.startswith('user ='):
return line.split('=', 1)[1].strip().replace('"', '')
def query_elasticsearch(query, auth):
def query_elasticsearch(query, auth, index):
url = f"https://localhost:9200/{index}/_search"
headers = {"Content-Type": "application/json"}
response = requests.get(ES_URL, headers=headers, data=query, auth=auth, verify=False)
response = requests.get(url, headers=headers, data=query, auth=auth, verify=False)
response.raise_for_status()
return response.json()
@@ -47,12 +49,12 @@ def save_content(hit, base_folder, subfolder="", extension="txt"):
f.write(content)
return file_path
def save_overrides(hit):
def save_overrides(hit, output_dir):
so_detection = hit["_source"]["so_detection"]
public_id = so_detection["publicId"]
overrides = so_detection["overrides"]
language = so_detection["language"]
folder = os.path.join(OUTPUT_DIR, language, "overrides")
folder = os.path.join(output_dir, language, "overrides")
os.makedirs(folder, exist_ok=True)
extension = "yaml" if language == "sigma" else "txt"
file_path = os.path.join(folder, f"{public_id}.{extension}")
@@ -60,20 +62,20 @@ def save_overrides(hit):
f.write('\n'.join(json.dumps(override) for override in overrides) if isinstance(overrides, list) else overrides)
return file_path
def ensure_git_repo():
if not os.path.isdir(os.path.join(OUTPUT_DIR, '.git')):
def ensure_git_repo(output_dir):
if not os.path.isdir(os.path.join(output_dir, '.git')):
subprocess.run(["git", "config", "--global", "init.defaultBranch", "main"], check=True)
subprocess.run(["git", "-C", OUTPUT_DIR, "init"], check=True)
subprocess.run(["git", "-C", OUTPUT_DIR, "remote", "add", "origin", "default"], check=True)
subprocess.run(["git", "-C", output_dir, "init"], check=True)
subprocess.run(["git", "-C", output_dir, "remote", "add", "origin", "default"], check=True)
def commit_changes():
ensure_git_repo()
subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.email", "securityonion@local.invalid"], check=True)
subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.name", "securityonion"], check=True)
subprocess.run(["git", "-C", OUTPUT_DIR, "add", "."], check=True)
status_result = subprocess.run(["git", "-C", OUTPUT_DIR, "status"], capture_output=True, text=True)
def commit_changes(output_dir):
ensure_git_repo(output_dir)
subprocess.run(["git", "-C", output_dir, "config", "user.email", "securityonion@local.invalid"], check=True)
subprocess.run(["git", "-C", output_dir, "config", "user.name", "securityonion"], check=True)
subprocess.run(["git", "-C", output_dir, "add", "."], check=True)
status_result = subprocess.run(["git", "-C", output_dir, "status"], capture_output=True, text=True)
print(status_result.stdout)
commit_result = subprocess.run(["git", "-C", OUTPUT_DIR, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True)
commit_result = subprocess.run(["git", "-C", output_dir, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True)
if commit_result.returncode == 1:
print("No changes to commit.")
elif commit_result.returncode == 0:
@@ -81,29 +83,41 @@ def commit_changes():
else:
commit_result.check_returncode()
def parse_args():
parser = argparse.ArgumentParser(description="Backup custom detections and overrides from Elasticsearch")
parser.add_argument("--output", "-o", default=DEFAULT_OUTPUT_DIR,
help=f"Output directory for backups (default: {DEFAULT_OUTPUT_DIR})")
parser.add_argument("--index", "-i", default=DEFAULT_INDEX,
help=f"Elasticsearch index to query (default: {DEFAULT_INDEX})")
return parser.parse_args()
def main():
args = parse_args()
output_dir = args.output
index = args.index
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"Backing up Custom Detections and all Overrides to {OUTPUT_DIR} - {timestamp}\n")
os.makedirs(OUTPUT_DIR, exist_ok=True)
print(f"Backing up Custom Detections and all Overrides to {output_dir} - {timestamp}\n")
os.makedirs(output_dir, exist_ok=True)
auth_credentials = get_auth_credentials(AUTH_FILE)
username, password = auth_credentials.split(':', 1)
auth = HTTPBasicAuth(username, password)
# Query and save custom detections
detections = query_elasticsearch(QUERY_DETECTIONS, auth)["hits"]["hits"]
detections = query_elasticsearch(QUERY_DETECTIONS, auth, index)["hits"]["hits"]
for hit in detections:
save_content(hit, OUTPUT_DIR, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt")
save_content(hit, output_dir, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt")
# Query and save overrides
overrides = query_elasticsearch(QUERY_OVERRIDES, auth)["hits"]["hits"]
overrides = query_elasticsearch(QUERY_OVERRIDES, auth, index)["hits"]["hits"]
for hit in overrides:
save_overrides(hit)
commit_changes()
save_overrides(hit, output_dir)
commit_changes(output_dir)
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"Backup Completed - {timestamp}")
except Exception as e:

56
salt/soc/files/soc/so-detections-backup_test.py
View File

@@ -57,12 +57,12 @@ class TestBackupScript(unittest.TestCase):
mock_response.json.return_value = {'hits': {'hits': []}}
mock_response.raise_for_status = MagicMock()
mock_get.return_value = mock_response
response = ds.query_elasticsearch(ds.QUERY_DETECTIONS, self.auth)
response = ds.query_elasticsearch(ds.QUERY_DETECTIONS, self.auth, ds.DEFAULT_INDEX)
self.assertEqual(response, {'hits': {'hits': []}})
mock_get.assert_called_once_with(
ds.ES_URL,
f"https://localhost:9200/{ds.DEFAULT_INDEX}/_search",
headers={"Content-Type": "application/json"},
data=ds.QUERY_DETECTIONS,
auth=self.auth,
@@ -81,7 +81,7 @@ class TestBackupScript(unittest.TestCase):
@patch('os.makedirs')
@patch('builtins.open', new_callable=mock_open)
def test_save_overrides(self, mock_file, mock_makedirs):
file_path = ds.save_overrides(self.mock_override_hit)
file_path = ds.save_overrides(self.mock_override_hit, self.output_dir)
expected_path = f'{self.output_dir}/sigma/overrides/test_id.yaml'
self.assertEqual(file_path, expected_path)
mock_makedirs.assert_called_once_with(f'{self.output_dir}/sigma/overrides', exist_ok=True)
@@ -90,9 +90,9 @@ class TestBackupScript(unittest.TestCase):
@patch('subprocess.run')
def test_ensure_git_repo(self, mock_run):
mock_run.return_value = MagicMock(returncode=0)
ds.ensure_git_repo()
ds.ensure_git_repo(self.output_dir)
mock_run.assert_has_calls([
call(["git", "config", "--global", "init.defaultBranch", "main"], check=True),
call(["git", "-C", self.output_dir, "init"], check=True),
@@ -106,9 +106,9 @@ class TestBackupScript(unittest.TestCase):
mock_commit_result = MagicMock(returncode=1)
# Ensure sufficient number of MagicMock instances for each subprocess.run call
mock_run.side_effect = [mock_status_result, mock_commit_result, MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0)]
print("Running test_commit_changes...")
ds.commit_changes()
ds.commit_changes(self.output_dir)
print("Finished test_commit_changes.")
mock_run.assert_has_calls([
@@ -120,39 +120,45 @@ class TestBackupScript(unittest.TestCase):
])
@patch('builtins.print')
@patch('so-detections-backup.commit_changes')
@patch('so-detections-backup.save_overrides')
@patch('so-detections-backup.save_content')
@patch('so-detections-backup.query_elasticsearch')
@patch('so-detections-backup.get_auth_credentials')
@patch.object(ds, 'commit_changes')
@patch.object(ds, 'save_overrides')
@patch.object(ds, 'save_content')
@patch.object(ds, 'query_elasticsearch')
@patch.object(ds, 'get_auth_credentials')
@patch('os.makedirs')
def test_main(self, mock_makedirs, mock_get_auth, mock_query, mock_save_content, mock_save_overrides, mock_commit, mock_print):
@patch.object(ds, 'parse_args')
def test_main(self, mock_parse_args, mock_makedirs, mock_get_auth, mock_query, mock_save_content, mock_save_overrides, mock_commit, mock_print):
mock_args = MagicMock()
mock_args.output = self.output_dir
mock_args.index = ds.DEFAULT_INDEX
mock_parse_args.return_value = mock_args
mock_get_auth.return_value = self.auth_credentials
mock_query.side_effect = [
{'hits': {'hits': [{"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}]}},
{'hits': {'hits': [{"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}}]}}
]
with patch('datetime.datetime') as mock_datetime:
mock_datetime.now.return_value.strftime.return_value = "2024-05-23 20:49:44"
ds.main()
mock_makedirs.assert_called_once_with(self.output_dir, exist_ok=True)
mock_get_auth.assert_called_once_with(ds.AUTH_FILE)
mock_query.assert_has_calls([
call(ds.QUERY_DETECTIONS, self.auth),
call(ds.QUERY_OVERRIDES, self.auth)
call(ds.QUERY_DETECTIONS, self.auth, ds.DEFAULT_INDEX),
call(ds.QUERY_OVERRIDES, self.auth, ds.DEFAULT_INDEX)
])
mock_save_content.assert_called_once_with(
{"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}},
self.output_dir,
"sigma",
{"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}},
self.output_dir,
"sigma",
"yaml"
)
mock_save_overrides.assert_called_once_with(
{"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}}
{"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}},
self.output_dir
)
mock_commit.assert_called_once()
mock_commit.assert_called_once_with(self.output_dir)
mock_print.assert_called()
if __name__ == '__main__':