mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-05-23 19:45:25 +02:00
Move onionconfig writes out of so-yaml
This commit is contained in:
Mike Reeves
Executable
+448
@@ -0,0 +1,448 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
"""
|
||||
so-config.py writes SOC/onionconfig settings to Postgres.
|
||||
|
||||
so-yaml.py remains a YAML file editor. Call this tool when a pillar-backed
|
||||
setting also needs to be reflected in the onionconfig database.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
PILLAR_ROOT = Path(os.environ.get("SO_CONFIG_PILLAR_ROOT", "/opt/so/saltstack/local/pillar"))
|
||||
DOCKER_CONTAINER = os.environ.get("SO_CONFIG_PG_CONTAINER", "so-postgres")
|
||||
PG_DATABASE = os.environ.get("SO_CONFIG_PG_DATABASE", "securityonion")
|
||||
PG_USER = os.environ.get("SO_CONFIG_PG_USER", "postgres")
|
||||
DEFAULT_USER_ID = os.environ.get("SO_CONFIG_USER_ID", "so-config")
|
||||
|
||||
EXCLUDE_BASENAMES = {
|
||||
"secrets.sls",
|
||||
"auth.sls",
|
||||
"top.sls",
|
||||
}
|
||||
EXCLUDE_PATH_FRAGMENTS = (
|
||||
"/elasticsearch/nodes.sls",
|
||||
"/redis/nodes.sls",
|
||||
"/kafka/nodes.sls",
|
||||
"/hypervisor/nodes.sls",
|
||||
"/logstash/nodes.sls",
|
||||
"/node_data/ips.sls",
|
||||
"/postgres/auth.sls",
|
||||
"/elasticsearch/auth.sls",
|
||||
"/kibana/secrets.sls",
|
||||
)
|
||||
|
||||
|
||||
class SkipPath(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def pg_str(value):
|
||||
if value is None:
|
||||
return "NULL"
|
||||
return "'" + str(value).replace("'", "''") + "'"
|
||||
|
||||
|
||||
def pg_jsonb(value):
|
||||
return pg_str(json.dumps(value)) + "::jsonb"
|
||||
|
||||
|
||||
def docker_psql(sql):
|
||||
proc = subprocess.run(
|
||||
["docker", "exec", "-i", DOCKER_CONTAINER,
|
||||
"psql", "-U", PG_USER, "-d", PG_DATABASE,
|
||||
"-tA", "-q", "-v", "ON_ERROR_STOP=1"],
|
||||
input=sql.encode(),
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=60,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr.decode(errors="replace"))
|
||||
raise RuntimeError(f"docker exec psql failed with rc={proc.returncode}")
|
||||
return proc.stdout.decode(errors="replace")
|
||||
|
||||
|
||||
def schema_ready():
|
||||
sql = """
|
||||
SELECT to_regclass('public.settings') IS NOT NULL
|
||||
AND to_regclass('public.audit_settings') IS NOT NULL;
|
||||
"""
|
||||
return docker_psql(sql).strip() == "t"
|
||||
|
||||
|
||||
def cmd_wait_schema(args):
|
||||
import time
|
||||
|
||||
deadline = time.time() + args.timeout
|
||||
while time.time() <= deadline:
|
||||
if schema_ready():
|
||||
return 0
|
||||
time.sleep(args.interval)
|
||||
print("so-config: onionconfig schema is not ready", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def upsert_setting(setting_id, value, *, node_id="", duplicated_from_id=None,
|
||||
user_id=DEFAULT_USER_ID, note=None):
|
||||
note = note or "so-config upsert"
|
||||
sql = f"""
|
||||
BEGIN;
|
||||
WITH old_row AS (
|
||||
SELECT value
|
||||
FROM settings
|
||||
WHERE setting_id = {pg_str(setting_id)}
|
||||
AND node_id = {pg_str(node_id)}
|
||||
FOR UPDATE
|
||||
),
|
||||
upserted AS (
|
||||
INSERT INTO settings (setting_id, value, duplicated_from_id, node_id)
|
||||
VALUES ({pg_str(setting_id)}, {pg_jsonb(value)}, {pg_str(duplicated_from_id)}, {pg_str(node_id)})
|
||||
ON CONFLICT (setting_id, node_id) DO UPDATE
|
||||
SET value = EXCLUDED.value,
|
||||
duplicated_from_id = EXCLUDED.duplicated_from_id
|
||||
RETURNING value
|
||||
)
|
||||
INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
|
||||
SELECT {pg_str(setting_id)},
|
||||
{pg_str(node_id)},
|
||||
{pg_str(user_id)},
|
||||
(SELECT value FROM old_row),
|
||||
(SELECT value FROM upserted),
|
||||
{pg_str(note)}
|
||||
WHERE NOT EXISTS (SELECT 1 FROM old_row)
|
||||
OR (SELECT value FROM old_row) IS DISTINCT FROM (SELECT value FROM upserted);
|
||||
COMMIT;
|
||||
"""
|
||||
docker_psql(sql)
|
||||
|
||||
|
||||
def delete_setting(setting_id, *, node_id="", user_id=DEFAULT_USER_ID, note=None):
|
||||
note = note or "so-config delete"
|
||||
sql = f"""
|
||||
BEGIN;
|
||||
WITH deleted AS (
|
||||
DELETE FROM settings
|
||||
WHERE setting_id = {pg_str(setting_id)}
|
||||
AND node_id = {pg_str(node_id)}
|
||||
RETURNING value
|
||||
)
|
||||
INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
|
||||
SELECT {pg_str(setting_id)}, {pg_str(node_id)}, {pg_str(user_id)}, value, NULL::jsonb, {pg_str(note)}
|
||||
FROM deleted;
|
||||
COMMIT;
|
||||
"""
|
||||
docker_psql(sql)
|
||||
|
||||
|
||||
def delete_setting_prefix(setting_id, *, node_id="", user_id=DEFAULT_USER_ID, note=None):
|
||||
if not setting_id:
|
||||
raise ValueError("setting_id prefix cannot be empty")
|
||||
note = note or "so-config delete-prefix"
|
||||
sql = f"""
|
||||
BEGIN;
|
||||
WITH deleted AS (
|
||||
DELETE FROM settings
|
||||
WHERE node_id = {pg_str(node_id)}
|
||||
AND (
|
||||
setting_id = {pg_str(setting_id)}
|
||||
OR substring(setting_id from 1 for char_length({pg_str(setting_id)}) + 1) = {pg_str(setting_id + ".")}
|
||||
)
|
||||
RETURNING setting_id, value
|
||||
)
|
||||
INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
|
||||
SELECT setting_id, {pg_str(node_id)}, {pg_str(user_id)}, value, NULL::jsonb, {pg_str(note)}
|
||||
FROM deleted;
|
||||
COMMIT;
|
||||
"""
|
||||
docker_psql(sql)
|
||||
|
||||
|
||||
def purge_node(node_id, *, user_id=DEFAULT_USER_ID, note=None):
|
||||
note = note or "so-config purge-node"
|
||||
sql = f"""
|
||||
BEGIN;
|
||||
WITH deleted AS (
|
||||
DELETE FROM settings
|
||||
WHERE node_id = {pg_str(node_id)}
|
||||
RETURNING setting_id, value
|
||||
)
|
||||
INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
|
||||
SELECT setting_id, {pg_str(node_id)}, {pg_str(user_id)}, value, NULL::jsonb, {pg_str(note)}
|
||||
FROM deleted;
|
||||
COMMIT;
|
||||
"""
|
||||
docker_psql(sql)
|
||||
|
||||
|
||||
def parse_value(value, value_file=None):
|
||||
if value_file:
|
||||
with open(value_file, "r") as fh:
|
||||
value = fh.read()
|
||||
parsed = yaml.safe_load(value)
|
||||
if parsed is None and value == "":
|
||||
return ""
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_yaml_file(path):
|
||||
with open(path, "rb") as fh:
|
||||
raw = fh.read()
|
||||
if b"{%" in raw or b"{{" in raw:
|
||||
raise SkipPath(f"{path}: Jinja-templated files stay disk-only")
|
||||
if not raw.strip():
|
||||
return {}
|
||||
parsed = yaml.safe_load(raw)
|
||||
return parsed if parsed is not None else {}
|
||||
|
||||
|
||||
def flatten(prefix, value):
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
child_id = f"{prefix}.{key}" if prefix else str(key)
|
||||
yield from flatten(child_id, child)
|
||||
else:
|
||||
yield prefix, value
|
||||
|
||||
|
||||
def classify_pillar_path(path):
|
||||
norm = Path(path).resolve()
|
||||
norm_str = str(norm)
|
||||
|
||||
if norm.name in EXCLUDE_BASENAMES:
|
||||
raise SkipPath(f"{path}: excluded basename")
|
||||
for fragment in EXCLUDE_PATH_FRAGMENTS:
|
||||
if fragment in norm_str:
|
||||
raise SkipPath(f"{path}: excluded path fragment {fragment}")
|
||||
if norm.suffix != ".sls":
|
||||
raise SkipPath(f"{path}: not an .sls file")
|
||||
|
||||
parent = norm.parent.name
|
||||
stem = norm.stem
|
||||
|
||||
if parent == "minions":
|
||||
if stem.startswith("adv_"):
|
||||
return {"kind": "advanced", "setting_id": "advanced", "node_id": stem[4:]}
|
||||
return {"kind": "normal", "node_id": stem}
|
||||
|
||||
section = parent
|
||||
if stem == f"soc_{section}":
|
||||
return {"kind": "normal", "node_id": ""}
|
||||
if stem == f"adv_{section}":
|
||||
return {"kind": "advanced", "setting_id": f"{section}.advanced", "node_id": ""}
|
||||
|
||||
raise SkipPath(f"{path}: not a SOC-managed pillar file")
|
||||
|
||||
|
||||
def import_pillar_file(path, *, user_id=DEFAULT_USER_ID, note=None):
|
||||
meta = classify_pillar_path(path)
|
||||
note = note or f"so-config import-file {path}"
|
||||
|
||||
if meta["kind"] == "advanced":
|
||||
with open(path, "r") as fh:
|
||||
upsert_setting(meta["setting_id"], fh.read(), node_id=meta["node_id"],
|
||||
user_id=user_id, note=note)
|
||||
return 1
|
||||
|
||||
data = parse_yaml_file(path)
|
||||
if not isinstance(data, dict):
|
||||
raise SkipPath(f"{path}: top-level YAML is not a map")
|
||||
|
||||
count = 0
|
||||
for setting_id, value in flatten("", data):
|
||||
upsert_setting(setting_id, value, node_id=meta["node_id"],
|
||||
user_id=user_id, note=note)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def iter_pillar_files(root):
|
||||
root = Path(root)
|
||||
if not root.is_dir():
|
||||
return
|
||||
for path in sorted(root.rglob("*.sls")):
|
||||
if path.is_file():
|
||||
yield path
|
||||
|
||||
|
||||
def cmd_set(args):
|
||||
upsert_setting(args.setting_id, parse_value(args.value, args.value_file),
|
||||
node_id=args.node_id,
|
||||
duplicated_from_id=args.duplicated_from_id,
|
||||
user_id=args.user_id,
|
||||
note=args.note)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_delete(args):
|
||||
delete_setting(args.setting_id, node_id=args.node_id,
|
||||
user_id=args.user_id, note=args.note)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_delete_prefix(args):
|
||||
delete_setting_prefix(args.setting_id, node_id=args.node_id,
|
||||
user_id=args.user_id, note=args.note)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_purge_node(args):
|
||||
purge_node(args.node_id, user_id=args.user_id, note=args.note)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_import_file(args):
|
||||
count = import_pillar_file(args.path, user_id=args.user_id, note=args.note)
|
||||
print(f"imported {count} settings from {args.path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_import_minion(args):
|
||||
count = 0
|
||||
for name in (f"{args.node_id}.sls", f"adv_{args.node_id}.sls"):
|
||||
path = PILLAR_ROOT / "minions" / name
|
||||
if path.exists():
|
||||
count += import_pillar_file(path, user_id=args.user_id, note=args.note)
|
||||
print(f"imported {count} settings for node {args.node_id}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_import_all(args):
|
||||
count = 0
|
||||
skipped = 0
|
||||
for path in iter_pillar_files(args.root):
|
||||
try:
|
||||
count += import_pillar_file(path, user_id=args.user_id, note=args.note)
|
||||
except SkipPath as exc:
|
||||
skipped += 1
|
||||
if args.verbose:
|
||||
print(f"skip: {exc}", file=sys.stderr)
|
||||
print(f"imported {count} settings, skipped {skipped} files")
|
||||
if args.state_file:
|
||||
with open(args.state_file, "w") as fh:
|
||||
fh.write("ok\n")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync_yaml_mutation(args):
|
||||
meta = classify_pillar_path(args.path)
|
||||
note = args.note or f"so-config sync-yaml-mutation {args.operation} {args.path}"
|
||||
|
||||
if meta["kind"] == "advanced":
|
||||
import_pillar_file(args.path, user_id=args.user_id, note=note)
|
||||
return 0
|
||||
|
||||
if args.operation in ("add", "replace"):
|
||||
upsert_setting(args.key, parse_value(args.value, args.value_file),
|
||||
node_id=meta["node_id"],
|
||||
user_id=args.user_id,
|
||||
note=note)
|
||||
elif args.operation == "remove":
|
||||
delete_setting_prefix(args.key, node_id=meta["node_id"],
|
||||
user_id=args.user_id, note=note)
|
||||
else:
|
||||
raise ValueError(f"unsupported operation: {args.operation}")
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
p = sub.add_parser("wait-schema", help="wait for SOC-created onionconfig tables")
|
||||
p.add_argument("--timeout", type=int, default=120)
|
||||
p.add_argument("--interval", type=int, default=2)
|
||||
p.set_defaults(func=cmd_wait_schema)
|
||||
|
||||
p = sub.add_parser("set", help="upsert one setting")
|
||||
p.add_argument("setting_id")
|
||||
p.add_argument("value", nargs="?", default="")
|
||||
p.add_argument("--value-file")
|
||||
p.add_argument("--node-id", default="")
|
||||
p.add_argument("--duplicated-from-id")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note")
|
||||
p.set_defaults(func=cmd_set)
|
||||
|
||||
p = sub.add_parser("delete", help="delete one setting")
|
||||
p.add_argument("setting_id")
|
||||
p.add_argument("--node-id", default="")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note")
|
||||
p.set_defaults(func=cmd_delete)
|
||||
|
||||
p = sub.add_parser("delete-prefix", help="delete one setting and all child settings")
|
||||
p.add_argument("setting_id")
|
||||
p.add_argument("--node-id", default="")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note")
|
||||
p.set_defaults(func=cmd_delete_prefix)
|
||||
|
||||
p = sub.add_parser("purge-node", help="delete all settings for one node")
|
||||
p.add_argument("node_id")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note")
|
||||
p.set_defaults(func=cmd_purge_node)
|
||||
|
||||
p = sub.add_parser("import-file", help="import one SOC-managed pillar file")
|
||||
p.add_argument("path")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note")
|
||||
p.set_defaults(func=cmd_import_file)
|
||||
|
||||
p = sub.add_parser("import-minion", help="import one minion's pillar files")
|
||||
p.add_argument("node_id")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note")
|
||||
p.set_defaults(func=cmd_import_minion)
|
||||
|
||||
p = sub.add_parser("import-all", help="import all SOC-managed local pillar files")
|
||||
p.add_argument("--root", default=str(PILLAR_ROOT))
|
||||
p.add_argument("--state-file")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note", default="so-config initial import")
|
||||
p.add_argument("--verbose", action="store_true")
|
||||
p.set_defaults(func=cmd_import_all)
|
||||
|
||||
p = sub.add_parser("sync-yaml-mutation",
|
||||
help="mirror one so-yaml add/replace/remove mutation to onionconfig")
|
||||
p.add_argument("path")
|
||||
p.add_argument("operation", choices=("add", "replace", "remove"))
|
||||
p.add_argument("key")
|
||||
p.add_argument("value", nargs="?", default="")
|
||||
p.add_argument("--value-file")
|
||||
p.add_argument("--user-id", default=DEFAULT_USER_ID)
|
||||
p.add_argument("--note")
|
||||
p.set_defaults(func=cmd_sync_yaml_mutation)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv):
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
return args.func(args)
|
||||
except SkipPath as exc:
|
||||
print(f"skip: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except Exception as exc:
|
||||
print(f"so-config: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,178 @@
|
||||
import importlib
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
soconfig = importlib.import_module("so-config")
|
||||
|
||||
|
||||
class TestSoConfigPathMapping(unittest.TestCase):
|
||||
|
||||
def test_classify_global_soc(self):
|
||||
meta = soconfig.classify_pillar_path(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(meta["kind"], "normal")
|
||||
self.assertEqual(meta["node_id"], "")
|
||||
|
||||
def test_classify_global_advanced(self):
|
||||
meta = soconfig.classify_pillar_path(
|
||||
"/opt/so/saltstack/local/pillar/soc/adv_soc.sls")
|
||||
self.assertEqual(meta["kind"], "advanced")
|
||||
self.assertEqual(meta["setting_id"], "soc.advanced")
|
||||
self.assertEqual(meta["node_id"], "")
|
||||
|
||||
def test_classify_minion(self):
|
||||
meta = soconfig.classify_pillar_path(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls")
|
||||
self.assertEqual(meta["kind"], "normal")
|
||||
self.assertEqual(meta["node_id"], "h1_sensor")
|
||||
|
||||
def test_classify_minion_advanced(self):
|
||||
meta = soconfig.classify_pillar_path(
|
||||
"/opt/so/saltstack/local/pillar/minions/adv_h1_sensor.sls")
|
||||
self.assertEqual(meta["kind"], "advanced")
|
||||
self.assertEqual(meta["setting_id"], "advanced")
|
||||
self.assertEqual(meta["node_id"], "h1_sensor")
|
||||
|
||||
def test_classify_skips_bootstrap(self):
|
||||
with self.assertRaises(soconfig.SkipPath):
|
||||
soconfig.classify_pillar_path(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls")
|
||||
|
||||
|
||||
class TestSoConfigImport(unittest.TestCase):
|
||||
|
||||
def test_flatten_keeps_lists_as_values(self):
|
||||
flattened = dict(soconfig.flatten("", {
|
||||
"host": {"mainip": "10.0.0.1"},
|
||||
"suricata": {"pcap": {"enabled": True}},
|
||||
"items": ["a", "b"],
|
||||
}))
|
||||
self.assertEqual(flattened["host.mainip"], "10.0.0.1")
|
||||
self.assertEqual(flattened["suricata.pcap.enabled"], True)
|
||||
self.assertEqual(flattened["items"], ["a", "b"])
|
||||
|
||||
def test_import_file_upserts_flattened_settings(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "h1_sensor.sls")
|
||||
minions = os.path.join(tmp, "minions")
|
||||
os.mkdir(minions)
|
||||
path = os.path.join(minions, "h1_sensor.sls")
|
||||
with open(path, "w") as fh:
|
||||
fh.write("host:\n mainip: 10.0.0.1\nsuricata:\n enabled: true\n")
|
||||
|
||||
calls = []
|
||||
with patch.object(soconfig, "upsert_setting",
|
||||
side_effect=lambda *args, **kwargs: calls.append((args, kwargs))):
|
||||
count = soconfig.import_pillar_file(path)
|
||||
|
||||
self.assertEqual(count, 2)
|
||||
self.assertIn((("host.mainip", "10.0.0.1"), {"node_id": "h1_sensor", "user_id": "so-config", "note": f"so-config import-file {path}"}), calls)
|
||||
self.assertIn((("suricata.enabled", True), {"node_id": "h1_sensor", "user_id": "so-config", "note": f"so-config import-file {path}"}), calls)
|
||||
|
||||
def test_import_advanced_file_upserts_raw_content(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
minions = os.path.join(tmp, "minions")
|
||||
os.mkdir(minions)
|
||||
path = os.path.join(minions, "adv_h1_sensor.sls")
|
||||
with open(path, "w") as fh:
|
||||
fh.write("custom:\n raw: true\n")
|
||||
|
||||
calls = []
|
||||
with patch.object(soconfig, "upsert_setting",
|
||||
side_effect=lambda *args, **kwargs: calls.append((args, kwargs))):
|
||||
count = soconfig.import_pillar_file(path)
|
||||
|
||||
self.assertEqual(count, 1)
|
||||
self.assertEqual(calls[0][0], ("advanced", "custom:\n raw: true\n"))
|
||||
self.assertEqual(calls[0][1]["node_id"], "h1_sensor")
|
||||
|
||||
|
||||
class TestSoConfigSql(unittest.TestCase):
|
||||
|
||||
def test_schema_ready_checks_soc_tables(self):
|
||||
captured = {}
|
||||
with patch.object(soconfig, "docker_psql",
|
||||
side_effect=lambda sql: captured.update({"sql": sql}) or "t\n"):
|
||||
ready = soconfig.schema_ready()
|
||||
|
||||
self.assertTrue(ready)
|
||||
self.assertIn("to_regclass('public.settings')", captured["sql"])
|
||||
self.assertIn("to_regclass('public.audit_settings')", captured["sql"])
|
||||
|
||||
def test_set_writes_settings_and_audit(self):
|
||||
captured = {}
|
||||
with patch.object(soconfig, "docker_psql",
|
||||
side_effect=lambda sql: captured.setdefault("sql", sql)):
|
||||
soconfig.upsert_setting("host.mainip", "10.0.0.1",
|
||||
node_id="h1_sensor", user_id="tester", note="unit")
|
||||
|
||||
self.assertIn("INSERT INTO settings", captured["sql"])
|
||||
self.assertIn("INSERT INTO audit_settings", captured["sql"])
|
||||
self.assertIn("'host.mainip'", captured["sql"])
|
||||
self.assertIn("'h1_sensor'", captured["sql"])
|
||||
self.assertIn("'tester'", captured["sql"])
|
||||
|
||||
def test_purge_node_audits_deleted_rows(self):
|
||||
captured = {}
|
||||
with patch.object(soconfig, "docker_psql",
|
||||
side_effect=lambda sql: captured.setdefault("sql", sql)):
|
||||
soconfig.purge_node("h1_sensor", user_id="tester", note="unit")
|
||||
|
||||
self.assertIn("DELETE FROM settings", captured["sql"])
|
||||
self.assertIn("WHERE node_id = 'h1_sensor'", captured["sql"])
|
||||
self.assertIn("INSERT INTO audit_settings", captured["sql"])
|
||||
|
||||
def test_delete_prefix_removes_children_and_audits(self):
|
||||
captured = {}
|
||||
with patch.object(soconfig, "docker_psql",
|
||||
side_effect=lambda sql: captured.setdefault("sql", sql)):
|
||||
soconfig.delete_setting_prefix("elasticfleet", node_id="h1_sensor",
|
||||
user_id="tester", note="unit")
|
||||
|
||||
self.assertIn("DELETE FROM settings", captured["sql"])
|
||||
self.assertIn("setting_id = 'elasticfleet'", captured["sql"])
|
||||
self.assertIn("'elasticfleet.'", captured["sql"])
|
||||
self.assertIn("INSERT INTO audit_settings", captured["sql"])
|
||||
|
||||
def test_sync_yaml_replace_uses_path_node_id(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
minions = os.path.join(tmp, "minions")
|
||||
os.mkdir(minions)
|
||||
path = os.path.join(minions, "h1_sensor.sls")
|
||||
open(path, "w").close()
|
||||
|
||||
calls = []
|
||||
args = soconfig.build_parser().parse_args([
|
||||
"sync-yaml-mutation", path, "replace", "suricata.enabled", "true"
|
||||
])
|
||||
with patch.object(soconfig, "upsert_setting",
|
||||
side_effect=lambda *a, **kw: calls.append((a, kw))):
|
||||
soconfig.cmd_sync_yaml_mutation(args)
|
||||
|
||||
self.assertEqual(calls[0][0], ("suricata.enabled", True))
|
||||
self.assertEqual(calls[0][1]["node_id"], "h1_sensor")
|
||||
|
||||
def test_sync_yaml_remove_deletes_prefix(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
minions = os.path.join(tmp, "minions")
|
||||
os.mkdir(minions)
|
||||
path = os.path.join(minions, "h1_sensor.sls")
|
||||
open(path, "w").close()
|
||||
|
||||
calls = []
|
||||
args = soconfig.build_parser().parse_args([
|
||||
"sync-yaml-mutation", path, "remove", "elasticfleet"
|
||||
])
|
||||
with patch.object(soconfig, "delete_setting_prefix",
|
||||
side_effect=lambda *a, **kw: calls.append((a, kw))):
|
||||
soconfig.cmd_sync_yaml_mutation(args)
|
||||
|
||||
self.assertEqual(calls[0][0], ("elasticfleet",))
|
||||
self.assertEqual(calls[0][1]["node_id"], "h1_sensor")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -314,6 +314,24 @@ EOSQL
|
||||
fi
|
||||
}
|
||||
|
||||
function sync_minion_config_to_db() {
|
||||
log "INFO" "Syncing minion config to onionconfig for $MINION_ID"
|
||||
/usr/sbin/so-config.py import-minion "$MINION_ID" --note "so-minion $OPERATION"
|
||||
if [ $? -ne 0 ]; then
|
||||
log "ERROR" "Failed to sync minion config to onionconfig for $MINION_ID"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function purge_minion_config_from_db() {
|
||||
log "INFO" "Purging minion config from onionconfig for $MINION_ID"
|
||||
/usr/sbin/so-config.py purge-node "$MINION_ID" --note "so-minion delete"
|
||||
if [ $? -ne 0 ]; then
|
||||
log "ERROR" "Failed to purge minion config from onionconfig for $MINION_ID"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Create the minion file
|
||||
function ensure_socore_ownership() {
|
||||
log "INFO" "Setting socore ownership on minion files"
|
||||
@@ -1088,6 +1106,10 @@ case "$OPERATION" in
|
||||
log "ERROR" "Failed to setup minion files for $MINION_ID"
|
||||
exit 1
|
||||
}
|
||||
sync_minion_config_to_db || {
|
||||
log "ERROR" "Failed to sync minion config to onionconfig for $MINION_ID"
|
||||
exit 1
|
||||
}
|
||||
updateMineAndApplyStates || {
|
||||
log "ERROR" "Failed to update mine and apply states for $MINION_ID"
|
||||
exit 1
|
||||
@@ -1108,12 +1130,20 @@ case "$OPERATION" in
|
||||
log "ERROR" "Failed to setup VM minion files for $MINION_ID"
|
||||
exit 1
|
||||
}
|
||||
sync_minion_config_to_db || {
|
||||
log "ERROR" "Failed to sync VM minion config to onionconfig for $MINION_ID"
|
||||
exit 1
|
||||
}
|
||||
log "INFO" "Successfully added VM minion $MINION_ID"
|
||||
;;
|
||||
|
||||
"delete")
|
||||
log "INFO" "Removing minion $MINION_ID"
|
||||
remove_postgres_telegraf_from_minion
|
||||
purge_minion_config_from_db || {
|
||||
log "ERROR" "Failed to purge minion config from onionconfig for $MINION_ID"
|
||||
exit 1
|
||||
}
|
||||
deleteMinionFiles || {
|
||||
log "ERROR" "Failed to delete minion files for $MINION_ID"
|
||||
exit 1
|
||||
|
||||
@@ -1,329 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
"""
|
||||
so-pillar-import — populate the so_pillar.* schema in so-postgres from the
|
||||
on-disk Salt pillar tree.
|
||||
|
||||
Reads /opt/so/saltstack/local/pillar/, decomposes each .sls file into a
|
||||
(scope, role|minion_id, pillar_path, data) tuple, and UPSERTs it into
|
||||
so_pillar.pillar_entry. Idempotent — re-running with no SLS edits produces
|
||||
no version bumps because the audit trigger only writes a row when data
|
||||
actually changes.
|
||||
|
||||
Bootstrap and mine-driven files are skipped (see EXCLUDE_BASENAMES /
|
||||
EXCLUDE_PREFIXES below). Files containing Jinja templates ({% or {{) are
|
||||
also skipped — those stay disk-authoritative and ext_pillar_first: False
|
||||
means they render before the PG overlay anyway.
|
||||
|
||||
All SQL goes through `docker exec so-postgres psql` so no separate DSN
|
||||
config is required at first-install time. Designed to be called by
|
||||
salt/postgres/schema_pillar.sls (initial seed) and by salt/manager/tools/
|
||||
sbin/so-minion (per-minion sync on add/delete).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
PILLAR_LOCAL_ROOT = Path("/opt/so/saltstack/local/pillar")
|
||||
PILLAR_DEFAULT_ROOT = Path("/opt/so/saltstack/default/pillar")
|
||||
DOCKER_CONTAINER = "so-postgres"
|
||||
PG_SUPERUSER = "postgres"
|
||||
PG_DATABASE = "securityonion"
|
||||
|
||||
# Files that must NEVER move to Postgres. These are read by Salt before
|
||||
# Postgres is reachable, or contain renderer-time computed values (mine, etc.).
|
||||
EXCLUDE_BASENAMES = {
|
||||
"secrets.sls",
|
||||
"auth.sls", # postgres/auth.sls bootstrap
|
||||
"top.sls",
|
||||
}
|
||||
# Filename prefixes to skip — these are renderer-time computed pillars
|
||||
# (Salt mine, file_exists guards, etc.) that have to stay on disk.
|
||||
EXCLUDE_PATH_FRAGMENTS = (
|
||||
"/elasticsearch/nodes.sls",
|
||||
"/redis/nodes.sls",
|
||||
"/kafka/nodes.sls",
|
||||
"/hypervisor/nodes.sls",
|
||||
"/logstash/nodes.sls",
|
||||
"/node_data/ips.sls",
|
||||
"/postgres/auth.sls",
|
||||
"/elasticsearch/auth.sls",
|
||||
"/kibana/secrets.sls",
|
||||
)
|
||||
|
||||
|
||||
def log(level, msg):
|
||||
print(f"[{level}] {msg}", file=sys.stderr)
|
||||
|
||||
|
||||
def is_jinja_templated(content_bytes):
|
||||
return b"{%" in content_bytes or b"{{" in content_bytes
|
||||
|
||||
|
||||
def classify(path):
|
||||
"""Return (scope, role_name, minion_id, pillar_path) for a pillar file
|
||||
or None to skip it. role_name is None for now — the importer leaves role
|
||||
membership to the so_pillar.minion trigger and the salt/auth reactor."""
|
||||
rel_str = str(path)
|
||||
if path.name in EXCLUDE_BASENAMES:
|
||||
return None
|
||||
for frag in EXCLUDE_PATH_FRAGMENTS:
|
||||
if frag in rel_str:
|
||||
return None
|
||||
|
||||
# /local/pillar/minions/<id>.sls or adv_<id>.sls
|
||||
if path.parent.name == "minions":
|
||||
stem = path.stem # filename without .sls
|
||||
if stem.startswith("adv_"):
|
||||
mid = stem[4:]
|
||||
return ("minion", None, mid, f"minions.adv_{mid}")
|
||||
return ("minion", None, stem, f"minions.{stem}")
|
||||
|
||||
# /local/pillar/<section>/<file>.sls
|
||||
if path.parent.parent == PILLAR_LOCAL_ROOT or path.parent.parent == PILLAR_DEFAULT_ROOT:
|
||||
section = path.parent.name
|
||||
stem = path.stem
|
||||
# Only soc_<section>.sls and adv_<section>.sls are SOC-managed pillar
|
||||
# surfaces. Other files (e.g. nodes.sls, auth.sls, *.token) are
|
||||
# either covered by EXCLUDE_PATH_FRAGMENTS or are bootstrap surfaces
|
||||
# we leave alone for now.
|
||||
if stem.startswith("soc_") or stem.startswith("adv_"):
|
||||
return ("global", None, None, f"{section}.{stem}")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_yaml_file(path):
|
||||
with open(path, "rb") as f:
|
||||
content = f.read()
|
||||
if not content.strip():
|
||||
return {}
|
||||
if is_jinja_templated(content):
|
||||
return None
|
||||
data = yaml.safe_load(content)
|
||||
if data is None:
|
||||
return {}
|
||||
if not isinstance(data, dict):
|
||||
return {"_raw": data}
|
||||
return data
|
||||
|
||||
|
||||
def derive_node_type(minion_id):
|
||||
"""Conventional Security Onion minion ids are <host>_<role>. Take the
|
||||
last underscore-delimited token as the canonical role suffix."""
|
||||
parts = minion_id.rsplit("_", 1)
|
||||
if len(parts) == 2:
|
||||
return parts[1]
|
||||
return None
|
||||
|
||||
|
||||
def docker_psql(sql, *, db=PG_DATABASE, user=PG_SUPERUSER, on_error_stop=True, capture=True):
|
||||
"""Run sql via docker exec ... psql. Returns stdout as str."""
|
||||
args = [
|
||||
"docker", "exec", "-i", DOCKER_CONTAINER,
|
||||
"psql", "-U", user, "-d", db, "-tA", "-q",
|
||||
]
|
||||
if on_error_stop:
|
||||
args += ["-v", "ON_ERROR_STOP=1"]
|
||||
proc = subprocess.run(
|
||||
args, input=sql.encode(),
|
||||
capture_output=capture, check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr.decode(errors="replace"))
|
||||
raise RuntimeError(f"docker exec psql failed (rc={proc.returncode})")
|
||||
return proc.stdout.decode(errors="replace")
|
||||
|
||||
|
||||
def upsert_minion(minion_id, node_type):
|
||||
sql = (
|
||||
"INSERT INTO so_pillar.minion (minion_id, node_type) "
|
||||
f"VALUES ({pg_str(minion_id)}, {pg_str(node_type) if node_type else 'NULL'}) "
|
||||
"ON CONFLICT (minion_id) DO UPDATE SET node_type = EXCLUDED.node_type;"
|
||||
)
|
||||
docker_psql(sql)
|
||||
|
||||
|
||||
def delete_minion(minion_id):
|
||||
"""CASCADE removes pillar_entry + role_member rows."""
|
||||
sql = f"DELETE FROM so_pillar.minion WHERE minion_id = {pg_str(minion_id)};"
|
||||
docker_psql(sql)
|
||||
|
||||
|
||||
def upsert_pillar_entry(scope, role_name, minion_id, pillar_path, data, reason):
|
||||
"""Insert or update the row keyed by the partial unique index that
|
||||
matches scope. Audit trigger handles history; versioning trigger bumps
|
||||
version only when data changes."""
|
||||
data_json = json.dumps(data)
|
||||
role_sql = pg_str(role_name) if role_name else "NULL"
|
||||
minion_sql = pg_str(minion_id) if minion_id else "NULL"
|
||||
reason_sql = pg_str(reason)
|
||||
|
||||
if scope == "global":
|
||||
conflict = "(pillar_path) WHERE scope='global'"
|
||||
elif scope == "role":
|
||||
conflict = "(role_name, pillar_path) WHERE scope='role'"
|
||||
elif scope == "minion":
|
||||
conflict = "(minion_id, pillar_path) WHERE scope='minion'"
|
||||
else:
|
||||
raise ValueError(f"unknown scope {scope!r}")
|
||||
|
||||
sql = (
|
||||
"BEGIN;\n"
|
||||
f"SELECT set_config('so_pillar.change_reason', {reason_sql}, true);\n"
|
||||
f"INSERT INTO so_pillar.pillar_entry "
|
||||
f"(scope, role_name, minion_id, pillar_path, data, change_reason) "
|
||||
f"VALUES ({pg_str(scope)}, {role_sql}, {minion_sql}, {pg_str(pillar_path)}, {pg_jsonb(data_json)}, {reason_sql}) "
|
||||
f"ON CONFLICT {conflict} DO UPDATE "
|
||||
f"SET data = EXCLUDED.data, change_reason = EXCLUDED.change_reason;\n"
|
||||
"COMMIT;\n"
|
||||
)
|
||||
docker_psql(sql)
|
||||
|
||||
|
||||
def pg_str(s):
|
||||
"""Escape a Python str for inclusion in literal SQL. Pillar content has
|
||||
already been validated as YAML; we just need standard SQL escaping."""
|
||||
if s is None:
|
||||
return "NULL"
|
||||
return "'" + str(s).replace("'", "''") + "'"
|
||||
|
||||
|
||||
def pg_jsonb(json_str):
|
||||
return pg_str(json_str) + "::jsonb"
|
||||
|
||||
|
||||
def walk_pillar_root(root, paths):
|
||||
if not root.is_dir():
|
||||
return
|
||||
for path in root.rglob("*.sls"):
|
||||
if path.is_file():
|
||||
paths.append(path)
|
||||
|
||||
|
||||
def import_minion(minion_id, node_type, dry_run, reason):
|
||||
"""Re-import every pillar file for a single minion."""
|
||||
if not minion_id:
|
||||
raise ValueError("minion_id required for --scope minion")
|
||||
|
||||
upsert_minion(minion_id, node_type)
|
||||
log("INFO", f"Upserted minion row {minion_id} (node_type={node_type})")
|
||||
|
||||
targets = [
|
||||
PILLAR_LOCAL_ROOT / "minions" / f"{minion_id}.sls",
|
||||
PILLAR_LOCAL_ROOT / "minions" / f"adv_{minion_id}.sls",
|
||||
]
|
||||
for path in targets:
|
||||
if not path.exists():
|
||||
log("INFO", f" (no file at {path})")
|
||||
continue
|
||||
klass = classify(path)
|
||||
if not klass:
|
||||
log("INFO", f" skip {path} (excluded)")
|
||||
continue
|
||||
scope, role, mid, pillar_path = klass
|
||||
data = parse_yaml_file(path)
|
||||
if data is None:
|
||||
log("WARN", f" skip {path} (Jinja-templated; stays disk-only)")
|
||||
continue
|
||||
if dry_run:
|
||||
log("DRY", f" would upsert {scope}/{pillar_path} = {len(json.dumps(data))} bytes")
|
||||
continue
|
||||
upsert_pillar_entry(scope, role, mid, pillar_path, data, reason)
|
||||
log("INFO", f" imported {scope}/{pillar_path}")
|
||||
|
||||
|
||||
def import_all(dry_run, reason):
|
||||
"""Walk the entire local pillar tree and import every eligible file."""
|
||||
paths = []
|
||||
walk_pillar_root(PILLAR_LOCAL_ROOT, paths)
|
||||
|
||||
imported = 0
|
||||
skipped = 0
|
||||
minions_seen = set()
|
||||
|
||||
for path in sorted(paths):
|
||||
klass = classify(path)
|
||||
if not klass:
|
||||
skipped += 1
|
||||
continue
|
||||
scope, role, minion_id, pillar_path = klass
|
||||
data = parse_yaml_file(path)
|
||||
if data is None:
|
||||
log("WARN", f"skip {path} (Jinja-templated; stays disk-only)")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if scope == "minion" and minion_id not in minions_seen:
|
||||
node_type = derive_node_type(minion_id)
|
||||
if not dry_run:
|
||||
upsert_minion(minion_id, node_type)
|
||||
minions_seen.add(minion_id)
|
||||
|
||||
if dry_run:
|
||||
log("DRY", f"would upsert {scope}/{pillar_path} ({len(json.dumps(data))} bytes)")
|
||||
else:
|
||||
upsert_pillar_entry(scope, role, minion_id, pillar_path, data, reason)
|
||||
log("INFO", f"imported {scope}/{pillar_path}")
|
||||
imported += 1
|
||||
|
||||
log("INFO", f"done: {imported} imported, {skipped} skipped")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--scope", choices=("global", "role", "minion", "all"), default="all")
|
||||
ap.add_argument("--minion-id")
|
||||
ap.add_argument("--node-type", help="override node_type for --scope minion (default: derived from minion_id)")
|
||||
ap.add_argument("--delete", action="store_true",
|
||||
help="With --scope minion, remove the minion row (and its pillar rows via CASCADE)")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--diff", action="store_true",
|
||||
help="(reserved) print structural diffs vs current DB content")
|
||||
ap.add_argument("--yes", action="store_true",
|
||||
help="Skip confirmation prompts (currently unused; reserved)")
|
||||
ap.add_argument("--reason", default="so-pillar-import",
|
||||
help="change_reason recorded in pillar_entry_history")
|
||||
args = ap.parse_args()
|
||||
|
||||
try:
|
||||
if args.scope == "minion":
|
||||
if not args.minion_id:
|
||||
ap.error("--minion-id required when --scope minion")
|
||||
if args.delete:
|
||||
if args.dry_run:
|
||||
log("DRY", f"would delete {args.minion_id}")
|
||||
else:
|
||||
delete_minion(args.minion_id)
|
||||
log("INFO", f"deleted {args.minion_id}")
|
||||
else:
|
||||
node_type = args.node_type or derive_node_type(args.minion_id)
|
||||
import_minion(args.minion_id, node_type, args.dry_run, args.reason)
|
||||
elif args.scope == "all":
|
||||
import_all(args.dry_run, args.reason)
|
||||
else:
|
||||
log("ERROR", f"--scope {args.scope} not yet implemented; use --scope all or --scope minion")
|
||||
return 2
|
||||
except Exception as e:
|
||||
log("ERROR", str(e))
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -13,64 +13,6 @@ import json
|
||||
|
||||
lockFile = "/tmp/so-yaml.lock"
|
||||
|
||||
# postsalt: so-yaml supports three backend modes for PG-managed pillar paths:
|
||||
#
|
||||
# dual — write disk + mirror to so_pillar.*. Reads from disk.
|
||||
# Used during the migration transition when disk is still
|
||||
# canonical and PG runs as a shadow.
|
||||
# postgres — write to so_pillar.* only. Reads from so_pillar.*. No disk
|
||||
# file is touched. The end state once cutover is complete.
|
||||
# disk — disk only, no PG. Emergency rollback escape hatch.
|
||||
#
|
||||
# Bootstrap and mine-driven files (secrets.sls, ca/init.sls, */nodes.sls,
|
||||
# top.sls, etc.) are always handled on disk regardless of mode — those paths
|
||||
# are explicitly excluded by so_yaml_postgres.locate() raising SkipPath.
|
||||
#
|
||||
# Mode resolution: SO_YAML_BACKEND env var, then /opt/so/conf/so-yaml/mode,
|
||||
# then default 'dual' (safe upgrade behavior — flipping to 'postgres' is
|
||||
# done by schema_pillar.sls after the schema is in place and the importer
|
||||
# has run at least once).
|
||||
|
||||
MODE_FILE = "/opt/so/conf/so-yaml/mode"
|
||||
VALID_MODES = ("dual", "postgres", "disk")
|
||||
DEFAULT_MODE = "dual"
|
||||
|
||||
try:
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import so_yaml_postgres
|
||||
_SO_YAML_PG_AVAILABLE = True
|
||||
except Exception as _exc:
|
||||
_SO_YAML_PG_AVAILABLE = False
|
||||
|
||||
|
||||
def _resolveBackendMode():
|
||||
env = os.environ.get("SO_YAML_BACKEND")
|
||||
if env and env in VALID_MODES:
|
||||
return env
|
||||
try:
|
||||
with open(MODE_FILE, "r") as fh:
|
||||
value = fh.read().strip()
|
||||
if value in VALID_MODES:
|
||||
return value
|
||||
except (IOError, OSError):
|
||||
pass
|
||||
return DEFAULT_MODE
|
||||
|
||||
|
||||
_BACKEND_MODE = _resolveBackendMode()
|
||||
|
||||
|
||||
def _isPgManaged(filename):
|
||||
"""True when so-yaml should route this file's reads/writes through
|
||||
so_pillar.*. False for bootstrap/mine-driven files that always live on
|
||||
disk, and for arbitrary YAML paths outside the pillar tree."""
|
||||
if not _SO_YAML_PG_AVAILABLE:
|
||||
return False
|
||||
try:
|
||||
return so_yaml_postgres.is_pg_managed(filename)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def showUsage(args):
|
||||
print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
|
||||
@@ -83,14 +25,9 @@ def showUsage(args):
|
||||
print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
|
||||
print(' remove - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
|
||||
print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
|
||||
print(' purge - Delete the YAML file from disk and remove its rows from so_pillar.* (no KEY arg).', file=sys.stderr)
|
||||
print(' purge - Delete the YAML file from disk (no KEY arg).', file=sys.stderr)
|
||||
print(' help - Prints this usage information.', file=sys.stderr)
|
||||
print('', file=sys.stderr)
|
||||
print(' Backend mode:', file=sys.stderr)
|
||||
print(' Resolved from $SO_YAML_BACKEND, then /opt/so/conf/so-yaml/mode, default "dual".', file=sys.stderr)
|
||||
print(' Valid values: dual | postgres | disk. Bootstrap pillar files (secrets, ca, *.nodes.sls)', file=sys.stderr)
|
||||
print(' are always handled on disk regardless of mode.', file=sys.stderr)
|
||||
print('', file=sys.stderr)
|
||||
print(' Where:', file=sys.stderr)
|
||||
print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
|
||||
print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
|
||||
@@ -103,24 +40,6 @@ def showUsage(args):
|
||||
|
||||
|
||||
def loadYaml(filename):
|
||||
"""Load a YAML file's content as a dict.
|
||||
|
||||
PG-canonical mode (`postgres`): for PG-managed paths, read from
|
||||
so_pillar.pillar_entry. A missing row is treated as an empty dict so
|
||||
that `replace`/`add` on a fresh path can populate it from scratch.
|
||||
|
||||
Other modes / non-PG-managed paths: read from disk as today.
|
||||
"""
|
||||
if _BACKEND_MODE == "postgres" and _isPgManaged(filename):
|
||||
try:
|
||||
data = so_yaml_postgres.read_yaml(filename)
|
||||
except so_yaml_postgres.SkipPath:
|
||||
data = None
|
||||
except Exception as e:
|
||||
print(f"so-yaml: pg read failed for {filename}: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return data if data is not None else {}
|
||||
|
||||
try:
|
||||
with open(filename, "r") as file:
|
||||
content = file.read()
|
||||
@@ -134,96 +53,20 @@ def loadYaml(filename):
|
||||
|
||||
|
||||
def writeYaml(filename, content):
|
||||
"""Persist `content` for `filename`.
|
||||
|
||||
PG-canonical mode + PG-managed path: write only to so_pillar.*. A PG
|
||||
failure is fatal (no disk fallback) — caller must retry.
|
||||
|
||||
Dual mode: write disk, then mirror to PG (failures are warnings).
|
||||
|
||||
Disk mode or non-PG-managed path: write disk only.
|
||||
"""
|
||||
if _BACKEND_MODE == "postgres" and _isPgManaged(filename):
|
||||
if not _SO_YAML_PG_AVAILABLE:
|
||||
print("so-yaml: PG-canonical mode requires so_yaml_postgres module", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
ok, msg = so_yaml_postgres.write_yaml(
|
||||
filename, content,
|
||||
reason="so-yaml " + " ".join(sys.argv[1:2]))
|
||||
if not ok:
|
||||
print(f"so-yaml: pg write failed for {filename}: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return None
|
||||
|
||||
file = open(filename, "w")
|
||||
result = yaml.safe_dump(content, file)
|
||||
file.close()
|
||||
|
||||
if _BACKEND_MODE == "dual":
|
||||
_mirrorToPostgres(filename, content)
|
||||
return result
|
||||
|
||||
|
||||
def _mirrorToPostgres(filename, content):
|
||||
"""Best-effort dual-write of a YAML mutation into so_pillar.*. Skips
|
||||
files outside the PG-managed pillar surface (secrets.sls,
|
||||
elasticsearch/nodes.sls, etc.) and silently degrades when so-postgres
|
||||
is unreachable. Disk write is canonical in dual mode; this never
|
||||
raises.
|
||||
|
||||
Only real PG failures (`pg write failed: ...`) are logged so the
|
||||
common cases (skipped path, postgres not running) don't pollute
|
||||
stderr."""
|
||||
if not _SO_YAML_PG_AVAILABLE:
|
||||
return
|
||||
try:
|
||||
ok, msg = so_yaml_postgres.write_yaml(filename, content,
|
||||
reason="so-yaml " + " ".join(sys.argv[1:2]))
|
||||
if not ok and msg.startswith("pg write failed"):
|
||||
print(f"so-yaml: {msg}", file=sys.stderr)
|
||||
except Exception as e: # pragma: no cover — defensive: never break disk write
|
||||
print(f"so-yaml: pg mirror exception: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def purgeFile(filename):
|
||||
"""Delete a YAML file from disk and remove the matching rows from
|
||||
so_pillar.*. Idempotent — missing file/row counts as success.
|
||||
|
||||
PG-canonical mode + PG-managed path: PG delete is canonical. If a stale
|
||||
disk file from the dual-write era happens to still exist, it's removed
|
||||
too as a cleanup courtesy. PG failure is fatal in this mode.
|
||||
|
||||
Dual / disk modes: remove disk first; PG cleanup is best-effort."""
|
||||
if _BACKEND_MODE == "postgres" and _isPgManaged(filename):
|
||||
if not _SO_YAML_PG_AVAILABLE:
|
||||
print("so-yaml: PG-canonical mode requires so_yaml_postgres module", file=sys.stderr)
|
||||
return 1
|
||||
ok, msg = so_yaml_postgres.purge_yaml(filename, reason="so-yaml purge")
|
||||
if not ok:
|
||||
print(f"so-yaml: pg purge failed for {filename}: {msg}", file=sys.stderr)
|
||||
return 1
|
||||
if os.path.exists(filename):
|
||||
try:
|
||||
os.remove(filename)
|
||||
except Exception as e:
|
||||
print(f"so-yaml: warn — could not remove stale disk file {filename}: {e}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
"""Delete a YAML file from disk. Idempotent; missing files are success."""
|
||||
if os.path.exists(filename):
|
||||
try:
|
||||
os.remove(filename)
|
||||
except Exception as e:
|
||||
print(f"Failed to remove {filename}: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if _BACKEND_MODE == "dual" and _SO_YAML_PG_AVAILABLE:
|
||||
try:
|
||||
ok, msg = so_yaml_postgres.purge_yaml(filename,
|
||||
reason="so-yaml purge")
|
||||
if not ok and msg.startswith("pg purge failed"):
|
||||
print(f"so-yaml: {msg}", file=sys.stderr)
|
||||
except Exception as e:
|
||||
print(f"so-yaml: pg purge exception: {e}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -543,10 +386,7 @@ def get(args):
|
||||
|
||||
|
||||
def purge(args):
|
||||
"""purge YAML_FILE — delete the file from disk and remove the matching
|
||||
rows from so_pillar.* in so-postgres. Used by so-minion's delete path
|
||||
(in place of `rm -f`) so the audit log captures the deletion and
|
||||
role_member rows get cleaned up via FK CASCADE on so_pillar.minion."""
|
||||
"""purge YAML_FILE - delete the file from disk."""
|
||||
if len(args) != 1:
|
||||
print('Missing filename arg', file=sys.stderr)
|
||||
showUsage(None)
|
||||
|
||||
@@ -1007,9 +1007,7 @@ class TestPurge(unittest.TestCase):
|
||||
filename = "/tmp/so-yaml_test_purge.yaml"
|
||||
with open(filename, "w") as f:
|
||||
f.write("key: value\n")
|
||||
# Disable PG mirror so the test doesn't shell out to docker.
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', False):
|
||||
rc = soyaml.purge([filename])
|
||||
rc = soyaml.purge([filename])
|
||||
self.assertEqual(rc, 0)
|
||||
import os as _os
|
||||
self.assertFalse(_os.path.exists(filename))
|
||||
@@ -1019,301 +1017,5 @@ class TestPurge(unittest.TestCase):
|
||||
import os as _os
|
||||
if _os.path.exists(filename):
|
||||
_os.remove(filename)
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', False):
|
||||
rc = soyaml.purge([filename])
|
||||
rc = soyaml.purge([filename])
|
||||
self.assertEqual(rc, 0)
|
||||
|
||||
|
||||
class TestSoYamlPostgres(unittest.TestCase):
|
||||
"""Tests the path-locator and write/purge contract of the dual-write
|
||||
backend module without actually contacting Postgres."""
|
||||
|
||||
def setUp(self):
|
||||
import importlib
|
||||
self.mod = importlib.import_module("so_yaml_postgres")
|
||||
|
||||
def test_locate_global_soc(self):
|
||||
scope, role, mid, path = self.mod.locate(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(scope, "global")
|
||||
self.assertIsNone(role)
|
||||
self.assertIsNone(mid)
|
||||
self.assertEqual(path, "soc.soc_soc")
|
||||
|
||||
def test_locate_global_advanced(self):
|
||||
scope, role, mid, path = self.mod.locate(
|
||||
"/opt/so/saltstack/local/pillar/soc/adv_soc.sls")
|
||||
self.assertEqual(scope, "global")
|
||||
self.assertEqual(path, "soc.adv_soc")
|
||||
|
||||
def test_locate_minion(self):
|
||||
scope, role, mid, path = self.mod.locate(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls")
|
||||
self.assertEqual(scope, "minion")
|
||||
self.assertEqual(mid, "h1_sensor")
|
||||
self.assertEqual(path, "minions.h1_sensor")
|
||||
|
||||
def test_locate_minion_advanced(self):
|
||||
scope, role, mid, path = self.mod.locate(
|
||||
"/opt/so/saltstack/local/pillar/minions/adv_h1_sensor.sls")
|
||||
self.assertEqual(scope, "minion")
|
||||
self.assertEqual(mid, "h1_sensor")
|
||||
self.assertEqual(path, "minions.adv_h1_sensor")
|
||||
|
||||
def test_locate_skip_secrets(self):
|
||||
with self.assertRaises(self.mod.SkipPath):
|
||||
self.mod.locate("/opt/so/saltstack/local/pillar/secrets.sls")
|
||||
|
||||
def test_locate_skip_postgres_auth(self):
|
||||
with self.assertRaises(self.mod.SkipPath):
|
||||
self.mod.locate("/opt/so/saltstack/local/pillar/postgres/auth.sls")
|
||||
|
||||
def test_locate_skip_mine_driven(self):
|
||||
with self.assertRaises(self.mod.SkipPath):
|
||||
self.mod.locate("/opt/so/saltstack/local/pillar/elasticsearch/nodes.sls")
|
||||
|
||||
def test_locate_skip_top(self):
|
||||
with self.assertRaises(self.mod.SkipPath):
|
||||
self.mod.locate("/opt/so/saltstack/local/pillar/top.sls")
|
||||
|
||||
def test_locate_skip_unrelated(self):
|
||||
with self.assertRaises(self.mod.SkipPath):
|
||||
self.mod.locate("/etc/hostname")
|
||||
|
||||
def test_pg_str_escapes(self):
|
||||
self.assertEqual(self.mod._pg_str("a'b"), "'a''b'")
|
||||
self.assertEqual(self.mod._pg_str(None), "NULL")
|
||||
|
||||
def test_conflict_target(self):
|
||||
self.assertIn("scope='global'", self.mod._conflict_target("global"))
|
||||
self.assertIn("scope='role'", self.mod._conflict_target("role"))
|
||||
self.assertIn("scope='minion'", self.mod._conflict_target("minion"))
|
||||
with self.assertRaises(ValueError):
|
||||
self.mod._conflict_target("bogus")
|
||||
|
||||
def test_write_yaml_skips_disk_only_path(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
ok, msg = self.mod.write_yaml(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls",
|
||||
{"secrets": {"foo": "bar"}})
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("disk-only", msg)
|
||||
|
||||
def test_write_yaml_unreachable(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=False):
|
||||
ok, msg = self.mod.write_yaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls",
|
||||
{"soc": {"foo": "bar"}})
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(msg, "postgres unreachable")
|
||||
|
||||
def test_is_pg_managed_true(self):
|
||||
self.assertTrue(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls"))
|
||||
self.assertTrue(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls"))
|
||||
|
||||
def test_is_pg_managed_false_for_bootstrap(self):
|
||||
self.assertFalse(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls"))
|
||||
self.assertFalse(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/postgres/auth.sls"))
|
||||
self.assertFalse(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/elasticsearch/nodes.sls"))
|
||||
|
||||
def test_read_yaml_unreachable(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=False):
|
||||
self.assertIsNone(self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls"))
|
||||
|
||||
def test_read_yaml_skips_disk_only(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with self.assertRaises(self.mod.SkipPath):
|
||||
self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls")
|
||||
|
||||
def test_read_yaml_returns_data(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with patch.object(self.mod, '_docker_psql',
|
||||
return_value='{"soc": {"foo": "bar"}}\n'):
|
||||
data = self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(data, {"soc": {"foo": "bar"}})
|
||||
|
||||
def test_read_yaml_returns_none_when_no_row(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with patch.object(self.mod, '_docker_psql', return_value=''):
|
||||
data = self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertIsNone(data)
|
||||
|
||||
def test_read_yaml_minion_query_shape(self):
|
||||
captured = {}
|
||||
|
||||
def fake_psql(sql):
|
||||
captured['sql'] = sql
|
||||
return '{"host": {"mainip": "10.0.0.1"}}'
|
||||
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with patch.object(self.mod, '_docker_psql', side_effect=fake_psql):
|
||||
data = self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls")
|
||||
self.assertEqual(data, {"host": {"mainip": "10.0.0.1"}})
|
||||
self.assertIn("scope='minion'", captured['sql'])
|
||||
self.assertIn("'h1_sensor'", captured['sql'])
|
||||
self.assertIn("'minions.h1_sensor'", captured['sql'])
|
||||
|
||||
def test_is_enabled_public_alias(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
self.assertTrue(self.mod.is_enabled())
|
||||
with patch.object(self.mod, '_is_enabled', return_value=False):
|
||||
self.assertFalse(self.mod.is_enabled())
|
||||
|
||||
|
||||
class TestSoYamlBackendMode(unittest.TestCase):
|
||||
"""Tests so-yaml's backend-mode resolution and PG-canonical routing
|
||||
for read/write/purge. The PG calls themselves are stubbed; what we're
|
||||
asserting is that the right backend is chosen for each (mode, path)
|
||||
combination."""
|
||||
|
||||
def test_resolve_mode_env_overrides_file(self):
|
||||
with patch.dict('os.environ', {'SO_YAML_BACKEND': 'postgres'}):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'postgres')
|
||||
with patch.dict('os.environ', {'SO_YAML_BACKEND': 'disk'}):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'disk')
|
||||
|
||||
def test_resolve_mode_invalid_env_falls_back(self):
|
||||
with patch.dict('os.environ', {'SO_YAML_BACKEND': 'garbage'}, clear=False):
|
||||
with patch('builtins.open', side_effect=IOError):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'dual')
|
||||
|
||||
def test_resolve_mode_default_dual(self):
|
||||
env = {k: v for k, v in __import__('os').environ.items()
|
||||
if k != 'SO_YAML_BACKEND'}
|
||||
with patch.dict('os.environ', env, clear=True):
|
||||
with patch('builtins.open', side_effect=IOError):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'dual')
|
||||
|
||||
def test_is_pg_managed_proxies(self):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
self.assertTrue(soyaml._isPgManaged(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls"))
|
||||
self.assertFalse(soyaml._isPgManaged(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls"))
|
||||
|
||||
def test_is_pg_managed_false_when_module_unavailable(self):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', False):
|
||||
self.assertFalse(soyaml._isPgManaged(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls"))
|
||||
|
||||
def test_load_yaml_postgres_mode_reads_pg(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'read_yaml',
|
||||
return_value={"a": 1}):
|
||||
result = soyaml.loadYaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(result, {"a": 1})
|
||||
|
||||
def test_load_yaml_postgres_mode_returns_empty_when_no_row(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'read_yaml',
|
||||
return_value=None):
|
||||
result = soyaml.loadYaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(result, {})
|
||||
|
||||
def test_load_yaml_postgres_mode_reads_disk_for_bootstrap(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
f.write("foo: bar\n")
|
||||
tmp = f.name
|
||||
try:
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres,
|
||||
'is_pg_managed', return_value=False):
|
||||
result = soyaml.loadYaml(tmp)
|
||||
self.assertEqual(result, {"foo": "bar"})
|
||||
finally:
|
||||
_os.unlink(tmp)
|
||||
|
||||
def test_write_yaml_postgres_mode_skips_disk(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
tmp = f.name
|
||||
_os.unlink(tmp)
|
||||
try:
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'write_yaml',
|
||||
return_value=(True, 'ok')) as mock_w:
|
||||
soyaml.writeYaml(tmp, {"x": 1})
|
||||
self.assertFalse(_os.path.exists(tmp))
|
||||
mock_w.assert_called_once()
|
||||
finally:
|
||||
if _os.path.exists(tmp):
|
||||
_os.unlink(tmp)
|
||||
|
||||
def test_write_yaml_postgres_mode_failure_is_fatal(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'write_yaml',
|
||||
return_value=(False, 'pg write failed: connection refused')):
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_err:
|
||||
soyaml.writeYaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls",
|
||||
{"x": 1})
|
||||
sysmock.assert_called_with(1)
|
||||
|
||||
def test_write_yaml_disk_mode_skips_pg(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
tmp = f.name
|
||||
try:
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'disk'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'write_yaml') as mock_w:
|
||||
soyaml.writeYaml(tmp, {"x": 1})
|
||||
mock_w.assert_not_called()
|
||||
with open(tmp) as f:
|
||||
self.assertIn('x: 1', f.read())
|
||||
finally:
|
||||
_os.unlink(tmp)
|
||||
|
||||
def test_purge_postgres_mode_calls_pg_only(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
tmp = f.name
|
||||
_os.unlink(tmp)
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'purge_yaml',
|
||||
return_value=(True, 'ok')) as mock_p:
|
||||
rc = soyaml.purgeFile(tmp)
|
||||
self.assertEqual(rc, 0)
|
||||
mock_p.assert_called_once()
|
||||
|
||||
def test_purge_postgres_mode_failure_returns_nonzero(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'purge_yaml',
|
||||
return_value=(False, 'pg purge failed: x')):
|
||||
with patch('sys.stderr', new=StringIO()):
|
||||
rc = soyaml.purgeFile(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls")
|
||||
self.assertEqual(rc, 1)
|
||||
|
||||
@@ -1,320 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
"""
|
||||
so_yaml_postgres — Postgres-backed dual-write helpers for so-yaml.py.
|
||||
|
||||
so-yaml.py writes YAML pillar files on disk; this module mirrors those
|
||||
writes into so_pillar.* in so-postgres so ext_pillar and the SOC
|
||||
PostgresConfigstore see the same data. During the postsalt transition
|
||||
disk is canonical; PG writes are best-effort and never fail the disk
|
||||
operation.
|
||||
|
||||
Connection: shells out to `docker exec so-postgres psql -U postgres -d
|
||||
securityonion`. Same pattern so-pillar-import uses; avoids needing a
|
||||
separate DSN config at install time. Performance is fine because so-yaml
|
||||
is invoked from infrequent code paths (setup scripts, so-minion,
|
||||
so-firewall); SOC's hot path uses the in-process pgxpool in
|
||||
PostgresConfigstore, not so-yaml.
|
||||
|
||||
Path-to-row mapping mirrors PostgresConfigstore.locateSetting in
|
||||
securityonion-soc:
|
||||
|
||||
/opt/so/saltstack/local/pillar/<section>/soc_<section>.sls
|
||||
-> scope=global, pillar_path=<section>.soc_<section>
|
||||
/opt/so/saltstack/local/pillar/<section>/adv_<section>.sls
|
||||
-> scope=global, pillar_path=<section>.adv_<section>
|
||||
/opt/so/saltstack/local/pillar/minions/<id>.sls
|
||||
-> scope=minion, minion_id=<id>, pillar_path=minions.<id>
|
||||
/opt/so/saltstack/local/pillar/minions/adv_<id>.sls
|
||||
-> scope=minion, minion_id=<id>, pillar_path=minions.adv_<id>
|
||||
|
||||
Files outside that mapping (notably secrets.sls, postgres/auth.sls,
|
||||
elasticsearch/nodes.sls, etc.) are skipped — they stay disk-only forever
|
||||
or render dynamically and don't belong in PG.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
DOCKER_CONTAINER = os.environ.get("SO_PILLAR_PG_CONTAINER", "so-postgres")
|
||||
PG_DATABASE = os.environ.get("SO_PILLAR_PG_DATABASE", "securityonion")
|
||||
PG_USER = os.environ.get("SO_PILLAR_PG_USER", "postgres")
|
||||
|
||||
# File paths whose mutations stay disk-only forever. Mirrors EXCLUDE_*
|
||||
# in so-pillar-import.
|
||||
DISK_ONLY_PATHS = (
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls",
|
||||
"/opt/so/saltstack/local/pillar/postgres/auth.sls",
|
||||
"/opt/so/saltstack/local/pillar/elasticsearch/auth.sls",
|
||||
"/opt/so/saltstack/local/pillar/kibana/secrets.sls",
|
||||
)
|
||||
DISK_ONLY_FRAGMENTS = (
|
||||
"/elasticsearch/nodes.sls",
|
||||
"/redis/nodes.sls",
|
||||
"/kafka/nodes.sls",
|
||||
"/hypervisor/nodes.sls",
|
||||
"/logstash/nodes.sls",
|
||||
"/node_data/ips.sls",
|
||||
"/top.sls",
|
||||
)
|
||||
|
||||
|
||||
class SkipPath(Exception):
|
||||
"""Raised when a file path is intentionally not mirrored to PG."""
|
||||
|
||||
|
||||
def is_enabled():
|
||||
"""Public alias for callers that want to probe PG reachability without
|
||||
relying on a leading-underscore private name."""
|
||||
return _is_enabled()
|
||||
|
||||
|
||||
def _is_enabled():
|
||||
"""PG dual-write only fires if so-postgres is reachable. Cheap probe.
|
||||
Returns True when docker exec succeeds, False otherwise. We never
|
||||
want a PG hiccup to fail a disk write on a manager whose Postgres is
|
||||
momentarily unreachable."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["docker", "exec", DOCKER_CONTAINER,
|
||||
"pg_isready", "-h", "127.0.0.1", "-U", PG_USER, "-q"],
|
||||
capture_output=True, timeout=5, check=False,
|
||||
)
|
||||
return proc.returncode == 0
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def locate(path):
|
||||
"""Translate a so-yaml file path to (scope, role_name, minion_id, pillar_path).
|
||||
Raises SkipPath when the file is not part of the PG-managed surface."""
|
||||
norm = os.path.normpath(path)
|
||||
|
||||
if norm in DISK_ONLY_PATHS:
|
||||
raise SkipPath(f"{path}: explicit disk-only allowlist")
|
||||
for frag in DISK_ONLY_FRAGMENTS:
|
||||
if frag in norm:
|
||||
raise SkipPath(f"{path}: matches disk-only fragment {frag}")
|
||||
|
||||
parent = os.path.basename(os.path.dirname(norm))
|
||||
grandparent = os.path.basename(os.path.dirname(os.path.dirname(norm)))
|
||||
name = os.path.basename(norm)
|
||||
if not name.endswith(".sls"):
|
||||
raise SkipPath(f"{path}: not a .sls file")
|
||||
stem = name[:-4]
|
||||
|
||||
if parent == "minions":
|
||||
if stem.startswith("adv_"):
|
||||
mid = stem[4:]
|
||||
return ("minion", None, mid, f"minions.adv_{mid}")
|
||||
return ("minion", None, stem, f"minions.{stem}")
|
||||
|
||||
# /local/pillar/<section>/<file>.sls
|
||||
if grandparent == "pillar" and parent and parent != "":
|
||||
if stem.startswith("soc_") or stem.startswith("adv_"):
|
||||
return ("global", None, None, f"{parent}.{stem}")
|
||||
raise SkipPath(f"{path}: <section>/{stem}.sls is not a soc_/adv_ file")
|
||||
|
||||
raise SkipPath(f"{path}: unrecognised pillar layout")
|
||||
|
||||
|
||||
def _pg_str(s):
|
||||
if s is None:
|
||||
return "NULL"
|
||||
return "'" + str(s).replace("'", "''") + "'"
|
||||
|
||||
|
||||
def _docker_psql(sql):
|
||||
"""Run sql via docker exec ... psql. Returns stdout. Caller catches
|
||||
exceptions and downgrades to a warning."""
|
||||
proc = subprocess.run(
|
||||
["docker", "exec", "-i", DOCKER_CONTAINER,
|
||||
"psql", "-U", PG_USER, "-d", PG_DATABASE,
|
||||
"-tA", "-q", "-v", "ON_ERROR_STOP=1"],
|
||||
input=sql.encode(), capture_output=True, check=False, timeout=30,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(proc.stderr.decode(errors="replace") or
|
||||
f"docker exec psql exit {proc.returncode}")
|
||||
return proc.stdout.decode(errors="replace")
|
||||
|
||||
|
||||
def _conflict_target(scope):
|
||||
if scope == "global":
|
||||
return "(pillar_path) WHERE scope='global'"
|
||||
if scope == "role":
|
||||
return "(role_name, pillar_path) WHERE scope='role'"
|
||||
if scope == "minion":
|
||||
return "(minion_id, pillar_path) WHERE scope='minion'"
|
||||
raise ValueError(f"unknown scope {scope!r}")
|
||||
|
||||
|
||||
def is_pg_managed(path):
|
||||
"""True if this path maps to a so_pillar.* row (locate() succeeds).
|
||||
Bootstrap and mine-driven files return False — they always live on
|
||||
disk regardless of so-yaml's backend mode."""
|
||||
try:
|
||||
locate(path)
|
||||
return True
|
||||
except SkipPath:
|
||||
return False
|
||||
|
||||
|
||||
def read_yaml(path):
|
||||
"""Return the content dict stored in so_pillar.pillar_entry for `path`,
|
||||
or None when no row exists. Raises SkipPath when `path` is not part of
|
||||
the PG-managed surface (caller should read disk in that case).
|
||||
|
||||
Used by so-yaml.py PG-canonical mode so `replace`, `get`, etc. resolve
|
||||
against the database rather than a stale (or absent) disk file."""
|
||||
if not _is_enabled():
|
||||
return None
|
||||
scope, role, minion_id, pillar_path = locate(path)
|
||||
|
||||
if scope == "minion":
|
||||
sql = ("SELECT data FROM so_pillar.pillar_entry "
|
||||
"WHERE scope='minion' "
|
||||
f"AND minion_id={_pg_str(minion_id)} "
|
||||
f"AND pillar_path={_pg_str(pillar_path)}")
|
||||
elif scope == "role":
|
||||
sql = ("SELECT data FROM so_pillar.pillar_entry "
|
||||
"WHERE scope='role' "
|
||||
f"AND role_name={_pg_str(role)} "
|
||||
f"AND pillar_path={_pg_str(pillar_path)}")
|
||||
else:
|
||||
sql = ("SELECT data FROM so_pillar.pillar_entry "
|
||||
"WHERE scope='global' "
|
||||
f"AND pillar_path={_pg_str(pillar_path)}")
|
||||
|
||||
try:
|
||||
out = _docker_psql(sql).strip()
|
||||
except Exception:
|
||||
return None
|
||||
if not out:
|
||||
return None
|
||||
try:
|
||||
return json.loads(out)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def write_yaml(path, content_dict, *, reason="so-yaml dual-write"):
|
||||
"""Mirror the disk write at `path` (whose content was just rendered as
|
||||
`content_dict`) into so_pillar.pillar_entry. Best-effort: any failure
|
||||
is swallowed so the caller (so-yaml.py) does not see it as a fatal."""
|
||||
if not _is_enabled():
|
||||
return False, "postgres unreachable"
|
||||
try:
|
||||
scope, role, minion_id, pillar_path = locate(path)
|
||||
except SkipPath as e:
|
||||
return False, str(e)
|
||||
|
||||
data_json = json.dumps(content_dict if content_dict is not None else {})
|
||||
role_sql = _pg_str(role)
|
||||
minion_sql = _pg_str(minion_id)
|
||||
reason_sql = _pg_str(reason)
|
||||
conflict = _conflict_target(scope)
|
||||
|
||||
sql_parts = []
|
||||
if scope == "minion":
|
||||
# FK requires the minion row before pillar_entry can reference it.
|
||||
sql_parts.append(
|
||||
f"INSERT INTO so_pillar.minion (minion_id) VALUES ({minion_sql}) "
|
||||
"ON CONFLICT (minion_id) DO NOTHING;"
|
||||
)
|
||||
sql_parts.append(
|
||||
"BEGIN;\n"
|
||||
f"SELECT set_config('so_pillar.change_reason', {reason_sql}, true);\n"
|
||||
"INSERT INTO so_pillar.pillar_entry "
|
||||
"(scope, role_name, minion_id, pillar_path, data, change_reason) "
|
||||
f"VALUES ({_pg_str(scope)}, {role_sql}, {minion_sql}, "
|
||||
f"{_pg_str(pillar_path)}, {_pg_str(data_json)}::jsonb, {reason_sql}) "
|
||||
f"ON CONFLICT {conflict} DO UPDATE "
|
||||
"SET data = EXCLUDED.data, change_reason = EXCLUDED.change_reason;\n"
|
||||
"COMMIT;\n"
|
||||
)
|
||||
|
||||
try:
|
||||
_docker_psql("\n".join(sql_parts))
|
||||
except Exception as e:
|
||||
return False, f"pg write failed: {e}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def purge_yaml(path, *, reason="so-yaml purge"):
|
||||
"""Mirror the disk file deletion at `path` by deleting the matching
|
||||
pillar_entry rows. For minion files also deletes the so_pillar.minion
|
||||
row (CASCADE removes pillar_entry + role_member rows)."""
|
||||
if not _is_enabled():
|
||||
return False, "postgres unreachable"
|
||||
try:
|
||||
scope, role, minion_id, pillar_path = locate(path)
|
||||
except SkipPath as e:
|
||||
return False, str(e)
|
||||
|
||||
reason_sql = _pg_str(reason)
|
||||
parts = ["BEGIN;",
|
||||
f"SELECT set_config('so_pillar.change_reason', {reason_sql}, true);"]
|
||||
|
||||
if scope == "minion":
|
||||
# If both <id>.sls and adv_<id>.sls are gone the trigger / CASCADE
|
||||
# cleans up role_member; otherwise we just remove this one row.
|
||||
parts.append(
|
||||
f"DELETE FROM so_pillar.pillar_entry "
|
||||
f"WHERE scope='minion' AND minion_id={_pg_str(minion_id)} "
|
||||
f"AND pillar_path={_pg_str(pillar_path)};"
|
||||
)
|
||||
parts.append(
|
||||
f"DELETE FROM so_pillar.minion WHERE minion_id={_pg_str(minion_id)} "
|
||||
"AND NOT EXISTS (SELECT 1 FROM so_pillar.pillar_entry "
|
||||
f"WHERE minion_id={_pg_str(minion_id)});"
|
||||
)
|
||||
else:
|
||||
parts.append(
|
||||
f"DELETE FROM so_pillar.pillar_entry "
|
||||
f"WHERE scope={_pg_str(scope)} AND pillar_path={_pg_str(pillar_path)};"
|
||||
)
|
||||
|
||||
parts.append("COMMIT;")
|
||||
|
||||
try:
|
||||
_docker_psql("\n".join(parts))
|
||||
except Exception as e:
|
||||
return False, f"pg purge failed: {e}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
# CLI for diagnostics. Not exercised by so-yaml.py itself.
|
||||
def _main(argv):
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("op", choices=("locate", "ping"))
|
||||
ap.add_argument("path", nargs="?")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.op == "ping":
|
||||
ok = _is_enabled()
|
||||
print("ok" if ok else "unreachable")
|
||||
return 0 if ok else 1
|
||||
if args.op == "locate":
|
||||
if not args.path:
|
||||
ap.error("locate requires PATH")
|
||||
try:
|
||||
scope, role, minion_id, pillar_path = locate(args.path)
|
||||
print(f"scope={scope} role={role} minion_id={minion_id} pillar_path={pillar_path}")
|
||||
return 0
|
||||
except SkipPath as e:
|
||||
print(f"SKIP: {e}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(_main(sys.argv[1:]))
|
||||
@@ -33,8 +33,11 @@ so-elastic-fleet-stop --force
|
||||
|
||||
status "Deleting Fleet Data from Pillars..."
|
||||
so-yaml.py remove /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls elasticfleet
|
||||
/usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls remove elasticfleet --note "so-elastic-fleet-reset"
|
||||
so-yaml.py remove /opt/so/saltstack/local/pillar/global/soc_global.sls global.fleet_grid_enrollment_token_general
|
||||
/usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/global/soc_global.sls remove global.fleet_grid_enrollment_token_general --note "so-elastic-fleet-reset"
|
||||
so-yaml.py remove /opt/so/saltstack/local/pillar/global/soc_global.sls global.fleet_grid_enrollment_token_heavy
|
||||
/usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/global/soc_global.sls remove global.fleet_grid_enrollment_token_heavy --note "so-elastic-fleet-reset"
|
||||
|
||||
status "Restarting Kibana..."
|
||||
so-kibana-restart --force
|
||||
|
||||
Reference in New Issue
Block a user