mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-05-10 21:30:30 +02:00
make so-yaml PG-canonical and add pillar-change reactor stack
Two coupled changes that together let so_pillar.* be the canonical config store, with config edits driving service reloads automatically: so-yaml PG-canonical mode - Adds /opt/so/conf/so-yaml/mode (and SO_YAML_BACKEND env override) with three values: dual (legacy), postgres (PG-only for managed paths), disk (emergency rollback). Bootstrap files (secrets.sls, ca/init.sls, *.nodes.sls, top.sls, ...) stay disk-only regardless via the existing SkipPath allowlist in so_yaml_postgres.locate. - loadYaml/writeYaml/purgeFile now route to so_pillar.* in postgres mode: replace/add/get all read+write the database with no disk file ever appearing. PG failure is fatal in postgres mode (no silent fallback); dual mode preserves the prior best-effort mirror. - so_yaml_postgres gains read_yaml(path), is_pg_managed(path), and is_enabled() so so-yaml can answer "is this path PG-managed and is PG up" without reaching into private helpers. - schema_pillar.sls writes /opt/so/conf/so-yaml/mode = postgres after the importer succeeds, so flipping postgres:so_pillar:enabled flips so-yaml's behavior in lockstep with the schema being live. pg_notify-driven change fan-out - 008_change_notify.sql adds so_pillar.change_queue + an AFTER trigger on pillar_entry that enqueues the locator and pg_notifies 'so_pillar_change'. Queue is drained at-least-once so engine restarts don't lose events; pg_notify is just the wakeup signal. - New salt-master engine pg_notify_pillar.py LISTENs on the channel, drains the queue with FOR UPDATE SKIP LOCKED, debounces bursts, and fires 'so/pillar/changed' events grouped by (scope, role, minion). - Reactor so_pillar_changed.sls catches the tag and dispatches to orch.so_pillar_reload, which carries a DISPATCH map of pillar-path prefix -> (state sls, role grain set) so adding a new service to the auto-reload list is a one-line edit instead of a new reactor. - Engine + reactor wiring is gated on the same postgres:so_pillar:enabled flag as the schema and ext_pillar config so the whole stack flips on/off together. Tests: 21 new cases (112 total, all passing) covering mode resolution, PG-managed detection, and PG-canonical read/write/purge routing with the PG client stubbed.
This commit is contained in:
Mike Reeves
@@ -13,10 +13,28 @@ import json
|
||||
|
||||
lockFile = "/tmp/so-yaml.lock"
|
||||
|
||||
# postsalt: dual-write each disk mutation into so_pillar.* in so-postgres so
|
||||
# Salt's ext_pillar and SOC's PostgresConfigstore see the same data without
|
||||
# requiring a separate writer. Failure of the PG side is logged but never
|
||||
# fails the disk write — disk is canonical during the migration transition.
|
||||
# postsalt: so-yaml supports three backend modes for PG-managed pillar paths:
|
||||
#
|
||||
# dual — write disk + mirror to so_pillar.*. Reads from disk.
|
||||
# Used during the migration transition when disk is still
|
||||
# canonical and PG runs as a shadow.
|
||||
# postgres — write to so_pillar.* only. Reads from so_pillar.*. No disk
|
||||
# file is touched. The end state once cutover is complete.
|
||||
# disk — disk only, no PG. Emergency rollback escape hatch.
|
||||
#
|
||||
# Bootstrap and mine-driven files (secrets.sls, ca/init.sls, */nodes.sls,
|
||||
# top.sls, etc.) are always handled on disk regardless of mode — those paths
|
||||
# are explicitly excluded by so_yaml_postgres.locate() raising SkipPath.
|
||||
#
|
||||
# Mode resolution: SO_YAML_BACKEND env var, then /opt/so/conf/so-yaml/mode,
|
||||
# then default 'dual' (safe upgrade behavior — flipping to 'postgres' is
|
||||
# done by schema_pillar.sls after the schema is in place and the importer
|
||||
# has run at least once).
|
||||
|
||||
MODE_FILE = "/opt/so/conf/so-yaml/mode"
|
||||
VALID_MODES = ("dual", "postgres", "disk")
|
||||
DEFAULT_MODE = "dual"
|
||||
|
||||
try:
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import so_yaml_postgres
|
||||
@@ -25,6 +43,35 @@ except Exception as _exc:
|
||||
_SO_YAML_PG_AVAILABLE = False
|
||||
|
||||
|
||||
def _resolveBackendMode():
|
||||
env = os.environ.get("SO_YAML_BACKEND")
|
||||
if env and env in VALID_MODES:
|
||||
return env
|
||||
try:
|
||||
with open(MODE_FILE, "r") as fh:
|
||||
value = fh.read().strip()
|
||||
if value in VALID_MODES:
|
||||
return value
|
||||
except (IOError, OSError):
|
||||
pass
|
||||
return DEFAULT_MODE
|
||||
|
||||
|
||||
_BACKEND_MODE = _resolveBackendMode()
|
||||
|
||||
|
||||
def _isPgManaged(filename):
|
||||
"""True when so-yaml should route this file's reads/writes through
|
||||
so_pillar.*. False for bootstrap/mine-driven files that always live on
|
||||
disk, and for arbitrary YAML paths outside the pillar tree."""
|
||||
if not _SO_YAML_PG_AVAILABLE:
|
||||
return False
|
||||
try:
|
||||
return so_yaml_postgres.is_pg_managed(filename)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def showUsage(args):
|
||||
print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
|
||||
print(' General commands:', file=sys.stderr)
|
||||
@@ -39,6 +86,11 @@ def showUsage(args):
|
||||
print(' purge - Delete the YAML file from disk and remove its rows from so_pillar.* (no KEY arg).', file=sys.stderr)
|
||||
print(' help - Prints this usage information.', file=sys.stderr)
|
||||
print('', file=sys.stderr)
|
||||
print(' Backend mode:', file=sys.stderr)
|
||||
print(' Resolved from $SO_YAML_BACKEND, then /opt/so/conf/so-yaml/mode, default "dual".', file=sys.stderr)
|
||||
print(' Valid values: dual | postgres | disk. Bootstrap pillar files (secrets, ca, *.nodes.sls)', file=sys.stderr)
|
||||
print(' are always handled on disk regardless of mode.', file=sys.stderr)
|
||||
print('', file=sys.stderr)
|
||||
print(' Where:', file=sys.stderr)
|
||||
print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
|
||||
print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
|
||||
@@ -51,6 +103,24 @@ def showUsage(args):
|
||||
|
||||
|
||||
def loadYaml(filename):
|
||||
"""Load a YAML file's content as a dict.
|
||||
|
||||
PG-canonical mode (`postgres`): for PG-managed paths, read from
|
||||
so_pillar.pillar_entry. A missing row is treated as an empty dict so
|
||||
that `replace`/`add` on a fresh path can populate it from scratch.
|
||||
|
||||
Other modes / non-PG-managed paths: read from disk as today.
|
||||
"""
|
||||
if _BACKEND_MODE == "postgres" and _isPgManaged(filename):
|
||||
try:
|
||||
data = so_yaml_postgres.read_yaml(filename)
|
||||
except so_yaml_postgres.SkipPath:
|
||||
data = None
|
||||
except Exception as e:
|
||||
print(f"so-yaml: pg read failed for {filename}: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return data if data is not None else {}
|
||||
|
||||
try:
|
||||
with open(filename, "r") as file:
|
||||
content = file.read()
|
||||
@@ -64,10 +134,33 @@ def loadYaml(filename):
|
||||
|
||||
|
||||
def writeYaml(filename, content):
|
||||
"""Persist `content` for `filename`.
|
||||
|
||||
PG-canonical mode + PG-managed path: write only to so_pillar.*. A PG
|
||||
failure is fatal (no disk fallback) — caller must retry.
|
||||
|
||||
Dual mode: write disk, then mirror to PG (failures are warnings).
|
||||
|
||||
Disk mode or non-PG-managed path: write disk only.
|
||||
"""
|
||||
if _BACKEND_MODE == "postgres" and _isPgManaged(filename):
|
||||
if not _SO_YAML_PG_AVAILABLE:
|
||||
print("so-yaml: PG-canonical mode requires so_yaml_postgres module", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
ok, msg = so_yaml_postgres.write_yaml(
|
||||
filename, content,
|
||||
reason="so-yaml " + " ".join(sys.argv[1:2]))
|
||||
if not ok:
|
||||
print(f"so-yaml: pg write failed for {filename}: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return None
|
||||
|
||||
file = open(filename, "w")
|
||||
result = yaml.safe_dump(content, file)
|
||||
file.close()
|
||||
_mirrorToPostgres(filename, content)
|
||||
|
||||
if _BACKEND_MODE == "dual":
|
||||
_mirrorToPostgres(filename, content)
|
||||
return result
|
||||
|
||||
|
||||
@@ -75,7 +168,8 @@ def _mirrorToPostgres(filename, content):
|
||||
"""Best-effort dual-write of a YAML mutation into so_pillar.*. Skips
|
||||
files outside the PG-managed pillar surface (secrets.sls,
|
||||
elasticsearch/nodes.sls, etc.) and silently degrades when so-postgres
|
||||
is unreachable. Disk write is canonical; this never raises.
|
||||
is unreachable. Disk write is canonical in dual mode; this never
|
||||
raises.
|
||||
|
||||
Only real PG failures (`pg write failed: ...`) are logged so the
|
||||
common cases (skipped path, postgres not running) don't pollute
|
||||
@@ -92,9 +186,29 @@ def _mirrorToPostgres(filename, content):
|
||||
|
||||
|
||||
def purgeFile(filename):
|
||||
"""Delete a YAML file from disk and mirror the deletion into PG.
|
||||
Idempotent: missing file → success. Mirrors so-yaml's other verbs
|
||||
in tolerating a soft PG failure."""
|
||||
"""Delete a YAML file from disk and remove the matching rows from
|
||||
so_pillar.*. Idempotent — missing file/row counts as success.
|
||||
|
||||
PG-canonical mode + PG-managed path: PG delete is canonical. If a stale
|
||||
disk file from the dual-write era happens to still exist, it's removed
|
||||
too as a cleanup courtesy. PG failure is fatal in this mode.
|
||||
|
||||
Dual / disk modes: remove disk first; PG cleanup is best-effort."""
|
||||
if _BACKEND_MODE == "postgres" and _isPgManaged(filename):
|
||||
if not _SO_YAML_PG_AVAILABLE:
|
||||
print("so-yaml: PG-canonical mode requires so_yaml_postgres module", file=sys.stderr)
|
||||
return 1
|
||||
ok, msg = so_yaml_postgres.purge_yaml(filename, reason="so-yaml purge")
|
||||
if not ok:
|
||||
print(f"so-yaml: pg purge failed for {filename}: {msg}", file=sys.stderr)
|
||||
return 1
|
||||
if os.path.exists(filename):
|
||||
try:
|
||||
os.remove(filename)
|
||||
except Exception as e:
|
||||
print(f"so-yaml: warn — could not remove stale disk file {filename}: {e}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if os.path.exists(filename):
|
||||
try:
|
||||
os.remove(filename)
|
||||
@@ -102,7 +216,7 @@ def purgeFile(filename):
|
||||
print(f"Failed to remove {filename}: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if _SO_YAML_PG_AVAILABLE:
|
||||
if _BACKEND_MODE == "dual" and _SO_YAML_PG_AVAILABLE:
|
||||
try:
|
||||
ok, msg = so_yaml_postgres.purge_yaml(filename,
|
||||
reason="so-yaml purge")
|
||||
|
||||
@@ -1106,3 +1106,214 @@ class TestSoYamlPostgres(unittest.TestCase):
|
||||
{"soc": {"foo": "bar"}})
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(msg, "postgres unreachable")
|
||||
|
||||
def test_is_pg_managed_true(self):
|
||||
self.assertTrue(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls"))
|
||||
self.assertTrue(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls"))
|
||||
|
||||
def test_is_pg_managed_false_for_bootstrap(self):
|
||||
self.assertFalse(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls"))
|
||||
self.assertFalse(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/postgres/auth.sls"))
|
||||
self.assertFalse(self.mod.is_pg_managed(
|
||||
"/opt/so/saltstack/local/pillar/elasticsearch/nodes.sls"))
|
||||
|
||||
def test_read_yaml_unreachable(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=False):
|
||||
self.assertIsNone(self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls"))
|
||||
|
||||
def test_read_yaml_skips_disk_only(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with self.assertRaises(self.mod.SkipPath):
|
||||
self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls")
|
||||
|
||||
def test_read_yaml_returns_data(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with patch.object(self.mod, '_docker_psql',
|
||||
return_value='{"soc": {"foo": "bar"}}\n'):
|
||||
data = self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(data, {"soc": {"foo": "bar"}})
|
||||
|
||||
def test_read_yaml_returns_none_when_no_row(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with patch.object(self.mod, '_docker_psql', return_value=''):
|
||||
data = self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertIsNone(data)
|
||||
|
||||
def test_read_yaml_minion_query_shape(self):
|
||||
captured = {}
|
||||
|
||||
def fake_psql(sql):
|
||||
captured['sql'] = sql
|
||||
return '{"host": {"mainip": "10.0.0.1"}}'
|
||||
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
with patch.object(self.mod, '_docker_psql', side_effect=fake_psql):
|
||||
data = self.mod.read_yaml(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls")
|
||||
self.assertEqual(data, {"host": {"mainip": "10.0.0.1"}})
|
||||
self.assertIn("scope='minion'", captured['sql'])
|
||||
self.assertIn("'h1_sensor'", captured['sql'])
|
||||
self.assertIn("'minions.h1_sensor'", captured['sql'])
|
||||
|
||||
def test_is_enabled_public_alias(self):
|
||||
with patch.object(self.mod, '_is_enabled', return_value=True):
|
||||
self.assertTrue(self.mod.is_enabled())
|
||||
with patch.object(self.mod, '_is_enabled', return_value=False):
|
||||
self.assertFalse(self.mod.is_enabled())
|
||||
|
||||
|
||||
class TestSoYamlBackendMode(unittest.TestCase):
|
||||
"""Tests so-yaml's backend-mode resolution and PG-canonical routing
|
||||
for read/write/purge. The PG calls themselves are stubbed; what we're
|
||||
asserting is that the right backend is chosen for each (mode, path)
|
||||
combination."""
|
||||
|
||||
def test_resolve_mode_env_overrides_file(self):
|
||||
with patch.dict('os.environ', {'SO_YAML_BACKEND': 'postgres'}):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'postgres')
|
||||
with patch.dict('os.environ', {'SO_YAML_BACKEND': 'disk'}):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'disk')
|
||||
|
||||
def test_resolve_mode_invalid_env_falls_back(self):
|
||||
with patch.dict('os.environ', {'SO_YAML_BACKEND': 'garbage'}, clear=False):
|
||||
with patch('builtins.open', side_effect=IOError):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'dual')
|
||||
|
||||
def test_resolve_mode_default_dual(self):
|
||||
env = {k: v for k, v in __import__('os').environ.items()
|
||||
if k != 'SO_YAML_BACKEND'}
|
||||
with patch.dict('os.environ', env, clear=True):
|
||||
with patch('builtins.open', side_effect=IOError):
|
||||
self.assertEqual(soyaml._resolveBackendMode(), 'dual')
|
||||
|
||||
def test_is_pg_managed_proxies(self):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
self.assertTrue(soyaml._isPgManaged(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls"))
|
||||
self.assertFalse(soyaml._isPgManaged(
|
||||
"/opt/so/saltstack/local/pillar/secrets.sls"))
|
||||
|
||||
def test_is_pg_managed_false_when_module_unavailable(self):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', False):
|
||||
self.assertFalse(soyaml._isPgManaged(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls"))
|
||||
|
||||
def test_load_yaml_postgres_mode_reads_pg(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'read_yaml',
|
||||
return_value={"a": 1}):
|
||||
result = soyaml.loadYaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(result, {"a": 1})
|
||||
|
||||
def test_load_yaml_postgres_mode_returns_empty_when_no_row(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'read_yaml',
|
||||
return_value=None):
|
||||
result = soyaml.loadYaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
|
||||
self.assertEqual(result, {})
|
||||
|
||||
def test_load_yaml_postgres_mode_reads_disk_for_bootstrap(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
f.write("foo: bar\n")
|
||||
tmp = f.name
|
||||
try:
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres,
|
||||
'is_pg_managed', return_value=False):
|
||||
result = soyaml.loadYaml(tmp)
|
||||
self.assertEqual(result, {"foo": "bar"})
|
||||
finally:
|
||||
_os.unlink(tmp)
|
||||
|
||||
def test_write_yaml_postgres_mode_skips_disk(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
tmp = f.name
|
||||
_os.unlink(tmp)
|
||||
try:
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'write_yaml',
|
||||
return_value=(True, 'ok')) as mock_w:
|
||||
soyaml.writeYaml(tmp, {"x": 1})
|
||||
self.assertFalse(_os.path.exists(tmp))
|
||||
mock_w.assert_called_once()
|
||||
finally:
|
||||
if _os.path.exists(tmp):
|
||||
_os.unlink(tmp)
|
||||
|
||||
def test_write_yaml_postgres_mode_failure_is_fatal(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'write_yaml',
|
||||
return_value=(False, 'pg write failed: connection refused')):
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_err:
|
||||
soyaml.writeYaml(
|
||||
"/opt/so/saltstack/local/pillar/soc/soc_soc.sls",
|
||||
{"x": 1})
|
||||
sysmock.assert_called_with(1)
|
||||
|
||||
def test_write_yaml_disk_mode_skips_pg(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
tmp = f.name
|
||||
try:
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'disk'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'write_yaml') as mock_w:
|
||||
soyaml.writeYaml(tmp, {"x": 1})
|
||||
mock_w.assert_not_called()
|
||||
with open(tmp) as f:
|
||||
self.assertIn('x: 1', f.read())
|
||||
finally:
|
||||
_os.unlink(tmp)
|
||||
|
||||
def test_purge_postgres_mode_calls_pg_only(self):
|
||||
import tempfile, os as _os
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
tmp = f.name
|
||||
_os.unlink(tmp)
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'purge_yaml',
|
||||
return_value=(True, 'ok')) as mock_p:
|
||||
rc = soyaml.purgeFile(tmp)
|
||||
self.assertEqual(rc, 0)
|
||||
mock_p.assert_called_once()
|
||||
|
||||
def test_purge_postgres_mode_failure_returns_nonzero(self):
|
||||
with patch.object(soyaml, '_BACKEND_MODE', 'postgres'):
|
||||
with patch.object(soyaml, '_SO_YAML_PG_AVAILABLE', True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'is_pg_managed',
|
||||
return_value=True):
|
||||
with patch.object(soyaml.so_yaml_postgres, 'purge_yaml',
|
||||
return_value=(False, 'pg purge failed: x')):
|
||||
with patch('sys.stderr', new=StringIO()):
|
||||
rc = soyaml.purgeFile(
|
||||
"/opt/so/saltstack/local/pillar/minions/h1_sensor.sls")
|
||||
self.assertEqual(rc, 1)
|
||||
|
||||
@@ -69,6 +69,12 @@ class SkipPath(Exception):
|
||||
"""Raised when a file path is intentionally not mirrored to PG."""
|
||||
|
||||
|
||||
def is_enabled():
|
||||
"""Public alias for callers that want to probe PG reachability without
|
||||
relying on a leading-underscore private name."""
|
||||
return _is_enabled()
|
||||
|
||||
|
||||
def _is_enabled():
|
||||
"""PG dual-write only fires if so-postgres is reachable. Cheap probe.
|
||||
Returns True when docker exec succeeds, False otherwise. We never
|
||||
@@ -149,6 +155,55 @@ def _conflict_target(scope):
|
||||
raise ValueError(f"unknown scope {scope!r}")
|
||||
|
||||
|
||||
def is_pg_managed(path):
|
||||
"""True if this path maps to a so_pillar.* row (locate() succeeds).
|
||||
Bootstrap and mine-driven files return False — they always live on
|
||||
disk regardless of so-yaml's backend mode."""
|
||||
try:
|
||||
locate(path)
|
||||
return True
|
||||
except SkipPath:
|
||||
return False
|
||||
|
||||
|
||||
def read_yaml(path):
|
||||
"""Return the content dict stored in so_pillar.pillar_entry for `path`,
|
||||
or None when no row exists. Raises SkipPath when `path` is not part of
|
||||
the PG-managed surface (caller should read disk in that case).
|
||||
|
||||
Used by so-yaml.py PG-canonical mode so `replace`, `get`, etc. resolve
|
||||
against the database rather than a stale (or absent) disk file."""
|
||||
if not _is_enabled():
|
||||
return None
|
||||
scope, role, minion_id, pillar_path = locate(path)
|
||||
|
||||
if scope == "minion":
|
||||
sql = ("SELECT data FROM so_pillar.pillar_entry "
|
||||
"WHERE scope='minion' "
|
||||
f"AND minion_id={_pg_str(minion_id)} "
|
||||
f"AND pillar_path={_pg_str(pillar_path)}")
|
||||
elif scope == "role":
|
||||
sql = ("SELECT data FROM so_pillar.pillar_entry "
|
||||
"WHERE scope='role' "
|
||||
f"AND role_name={_pg_str(role)} "
|
||||
f"AND pillar_path={_pg_str(pillar_path)}")
|
||||
else:
|
||||
sql = ("SELECT data FROM so_pillar.pillar_entry "
|
||||
"WHERE scope='global' "
|
||||
f"AND pillar_path={_pg_str(pillar_path)}")
|
||||
|
||||
try:
|
||||
out = _docker_psql(sql).strip()
|
||||
except Exception:
|
||||
return None
|
||||
if not out:
|
||||
return None
|
||||
try:
|
||||
return json.loads(out)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def write_yaml(path, content_dict, *, reason="so-yaml dual-write"):
|
||||
"""Mirror the disk write at `path` (whose content was just rendered as
|
||||
`content_dict`) into so_pillar.pillar_entry. Best-effort: any failure
|
||||
|
||||
Reference in New Issue
Block a user