From 12f44478754205f5dc1128958a5f790d6ed2ed79 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 26 Jun 2026 15:40:32 -0400 Subject: [PATCH] Replace inotify rule-watch beacon with poll-based rules_db beacon Salt's stock inotify beacon leaks one kernel inotify instance every time the minion rebuilds the beacon loader's __context__ (the orphaned pyinotify.Notifier is never stopped), accumulating against fs.inotify.max_user_instances=128 until inotify_init() fails with EMFILE and rule-change push detection silently stops. This is independent of disable_during_state_run. Add a custom poll-based beacon (salt/_beacons/rules_db.py) modeled on pillar_db.py: it fingerprints the suricata/strelka rule dirs each interval (relpath + mtime_ns + size, temp files excluded) against a per-dir watermark, emitting an event only on change. It holds zero inotify instances, so the leak is impossible, and it keeps firing during state runs. Swap the inotify beacon config and reactor tag mappings accordingly; the push_suricata/push_strelka reactors are unchanged (they read only data['path']). --- salt/_beacons/rules_db.py | 139 ++++++++++++++++++ .../files/beacons_pushstate.conf.jinja | 40 +---- salt/reactor/push_strelka.sls | 4 +- salt/reactor/push_suricata.sls | 4 +- salt/salt/files/reactor_pushstate.conf | 8 +- 5 files changed, 150 insertions(+), 45 deletions(-) create mode 100644 salt/_beacons/rules_db.py diff --git a/salt/_beacons/rules_db.py b/salt/_beacons/rules_db.py new file mode 100644 index 000000000..ab63da431 --- /dev/null +++ b/salt/_beacons/rules_db.py @@ -0,0 +1,139 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Custom salt beacon that watches the suricata/strelka rule directories for changes +# and emits a beacon event per changed directory. This replaces the stock salt +# `inotify` beacon, which leaks a kernel inotify instance every time the minion +# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier +# without closing it) until fs.inotify.max_user_instances is exhausted and the +# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is +# impossible, and it keeps firing during state runs (no blackout). +# +# Detection is poll-based with a per-directory fingerprint persisted to +# WATERMARK_DIR: each pass walks the directory and hashes every file's +# (relpath, st_mtime_ns, st_size), which catches content writes, additions, +# moves, and deletions. A change in the digest emits one event; an unchanged +# digest emits nothing. This makes it self-healing (a missed poll simply catches +# up on the next one). +# +# Each emitted event carries the watched directory path under the configured tag +# (e.g. salt/beacon//rules_db/suricata); the push_suricata / push_strelka +# reactors write a push intent, after which the existing so-push-drainer / +# orch.push_batch pipeline takes over unchanged. + +import hashlib +import logging +import os +import re + +log = logging.getLogger(__name__) + +WATERMARK_DIR = '/opt/so/state' + +# Temp/editor files that should not trigger a push. Mirrors the exclude regexes +# the inotify beacon used. Matched against the full pathname. +EXCLUDES = [ + re.compile(r'\.sw[a-z]$'), + re.compile(r'~$'), + re.compile(r'/4913$'), + re.compile(r'/\.#'), +] + + +def __virtual__(): + return True + + +def validate(config): + return True, 'valid' + + +def _paths_from_config(config): + # The beacon config arrives as a list of single-key dicts (salt beacon style). + # Merge it and return the {dir: tag} mapping under the 'paths' key. + merged = {} + if isinstance(config, list): + for item in config: + if isinstance(item, dict): + merged.update(item) + elif isinstance(config, dict): + merged = config + paths = merged.get('paths', {}) + return paths if isinstance(paths, dict) else {} + + +def _excluded(pathname): + for pattern in EXCLUDES: + if pattern.search(pathname): + return True + return False + + +def _fingerprint(directory): + # Stat-only walk; hash each file's (relpath, mtime_ns, size). Returns a hex + # digest, or the digest of an empty tree if the directory does not exist. + h = hashlib.sha1() + if os.path.isdir(directory): + entries = [] + for root, _dirs, files in os.walk(directory): + for name in files: + full = os.path.join(root, name) + if _excluded(full): + continue + try: + st = os.stat(full) + except OSError: + continue + rel = os.path.relpath(full, directory) + entries.append('%s\0%d\0%d' % (rel, st.st_mtime_ns, st.st_size)) + for line in sorted(entries): + h.update(line.encode('utf-8', 'surrogateescape')) + h.update(b'\n') + return h.hexdigest() + + +def _watermark_file(tag): + return os.path.join(WATERMARK_DIR, 'rules_db_%s.hash' % tag) + + +def _read_watermark(tag): + try: + with open(_watermark_file(tag), 'r') as f: + return (f.read() or '').strip() or None + except IOError: + return None + + +def _write_watermark(tag, digest): + path = _watermark_file(tag) + try: + os.makedirs(WATERMARK_DIR, exist_ok=True) + tmp = path + '.tmp' + with open(tmp, 'w') as f: + f.write(digest) + os.rename(tmp, path) + except OSError: + log.exception('rules_db beacon: failed to persist watermark to %s', path) + + +def beacon(config): + retval = [] + + for directory, tag in _paths_from_config(config).items(): + digest = _fingerprint(directory) + previous = _read_watermark(tag) + + # First run / missing watermark: seed the digest and emit nothing so a + # fresh host does not fire a spurious fleetwide push. + if previous is None: + _write_watermark(tag, digest) + continue + + if digest != previous: + _write_watermark(tag, digest) + retval.append({'tag': tag, 'path': directory}) + log.info('rules_db beacon: change detected in %s, emitting %s', directory, tag) + + return retval diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index 4de5ef5e0..2c00163e3 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -3,39 +3,9 @@ beacons: pillar_db: - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: False - inotify: + rules_db: + - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: False - - coalesce: True - - files: - /opt/so/saltstack/local/salt/suricata/rules: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True - exclude: - - '\.sw[a-z]$': - regex: True - - '~$': - regex: True - - '/4913$': - regex: True - - '/\.#': - regex: True - /opt/so/saltstack/local/salt/strelka/rules/compiled: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True - exclude: - - '\.sw[a-z]$': - regex: True - - '~$': - regex: True - - '/4913$': - regex: True - - '/\.#': - regex: True + - paths: + /opt/so/saltstack/local/salt/suricata/rules: suricata + /opt/so/saltstack/local/salt/strelka/rules/compiled: strelka diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls index 21727bc91..d1d0207eb 100644 --- a/salt/reactor/push_strelka.sls +++ b/salt/reactor/push_strelka.sls @@ -1,7 +1,7 @@ #!py -# Reactor invoked by the inotify beacon on rule file changes under -# /opt/so/saltstack/local/salt/strelka/rules/compiled/. +# Reactor invoked by the rules_db poll beacon (salt/_beacons/rules_db.py) on rule +# file changes under /opt/so/saltstack/local/salt/strelka/rules/compiled/. # # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls index 53900e469..f50a92527 100644 --- a/salt/reactor/push_suricata.sls +++ b/salt/reactor/push_suricata.sls @@ -1,7 +1,7 @@ #!py -# Reactor invoked by the inotify beacon on rule file changes under -# /opt/so/saltstack/local/salt/suricata/rules/. +# Reactor invoked by the rules_db poll beacon (salt/_beacons/rules_db.py) on rule +# file changes under /opt/so/saltstack/local/salt/suricata/rules/. # # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes diff --git a/salt/salt/files/reactor_pushstate.conf b/salt/salt/files/reactor_pushstate.conf index 991c4d516..b4543b1a7 100644 --- a/salt/salt/files/reactor_pushstate.conf +++ b/salt/salt/files/reactor_pushstate.conf @@ -1,11 +1,7 @@ reactor: - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules': + - 'salt/beacon/*/rules_db/suricata': - salt://reactor/push_suricata.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules/*': - - salt://reactor/push_suricata.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled': - - salt://reactor/push_strelka.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/*': + - 'salt/beacon/*/rules_db/strelka': - salt://reactor/push_strelka.sls - 'salt/beacon/*/pillar_db/audit_settings': - salt://reactor/push_pillar.sls