From ec220572f0bf511ae3b95991ae00a5fba8aec4ef Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:32:06 -0600 Subject: [PATCH 01/17] dedup artifact registry config & split ea artifacts out of /opt/socore/html/ into dedicated /opt/socore/elastic-artifacts/artifacts/ --- salt/nginx/enabled.sls | 4 ++-- salt/nginx/etc/nginx.conf | 49 ++++++++++++--------------------------- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index cf3cb1e81..f1a71238a 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -54,9 +54,9 @@ so-nginx: - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages - - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts + - /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro {% if GLOBALS.is_manager %} + - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro # ATT&CK Navigator binds diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 357fc7e6d..5f9f9858d 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -4,6 +4,8 @@ {%- set role = grains.id.split('_') | last %} {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %} {%- set influxauth = ('so:' + influxpass) | base64_encode %} +{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %} +{%- set full_server_name = all_names | unique | join(' ') %} worker_processes auto; error_log /var/log/nginx/error.log; @@ -39,52 +41,31 @@ http { include /etc/nginx/conf.d/*.conf; - {%- if role in ['fleet'] %} - - server { - listen 8443; - server_name {{ GLOBALS.hostname }}; - root /opt/socore/html; - location /artifacts/ { - try_files $uri =206; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header X-Forwarded-Proto $scheme; - } - } - - {%- endif %} - {%- if GLOBALS.is_manager %} - {%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %} - {%- set full_server_name = all_names | unique | join(' ') %} - server { listen 80 default_server; server_name _; return 307 https://{{ GLOBALS.url_base }}$request_uri; } - + {%- endif %} + {%- if GLOBALS.is_manager or role == 'fleet' %} server { listen 8443; + {%- if GLOBALS.is_manager %} server_name {{ full_server_name }}; - root /opt/socore/html; + {%- else %} + server_name {{ GLOBALS.hostname }}; + {%- endif %} + root /opt/socore/elastic-artifacts; location /artifacts/ { - try_files $uri =206; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header X-Forwarded-Proto $scheme; + try_files $uri =404; + } + location / { + return 404; } } - + {%- endif %} + {%- if GLOBALS.is_manager %} server { listen 443 ssl default_server; http2 on; From 4a5253f68802fc9303d6c7dd21b2856e6f53a6ed Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 10 Nov 2026 08:26:19 -0500 Subject: [PATCH 02/17] Telegraf: fix EPS double counting and duplicate metric rows --- salt/telegraf/etc/telegraf.conf | 32 ++++++++++++++----- salt/telegraf/map.jinja | 2 +- salt/telegraf/scripts/checkfiles.sh | 2 +- salt/telegraf/scripts/containers.sh | 5 +-- salt/telegraf/scripts/eps.sh | 49 ++++++++++++++++++----------- salt/telegraf/scripts/features.sh | 3 +- 6 files changed, 62 insertions(+), 31 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index ab51ac2a5..0b2bb9bb6 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -138,6 +138,10 @@ # AGGREGATOR PLUGINS # ############################################################################### +[[aggregators.merge]] + drop_original = true + namepass = ["system", "swap", "kafka_partition"] + ############################################################################### # INPUT PLUGINS # ############################################################################### @@ -289,7 +293,7 @@ password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" {%- endif %} -{% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} +{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} [[inputs.jolokia2_agent]] name_prefix= "kafka_" urls = ["http://{{ NODEIP }}:8778/jolokia"] @@ -346,6 +350,26 @@ interval = "60s" {%- endif %} +{#- These report state that changes slowly; collecting them every interval only stores duplicate rows #} +{%- set SLOWSCRIPTS = [] %} +{%- for script in ['agentstatus.sh', 'os.sh', 'features.sh', 'raid.sh', 'zeekloss.sh'] %} +{%- if script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove(script) %} +{%- do SLOWSCRIPTS.append(script) %} +{%- endif %} +{%- endfor %} +{%- if SLOWSCRIPTS | length > 0 %} +[[inputs.exec]] + commands = [ +{%- for script in SLOWSCRIPTS %} + ["/scripts/{{script}}"]{% if not loop.last %},{% endif %} +{%- endfor %} + ] + data_format = "influx" + timeout = "15s" + interval = "5m" +{%- endif %} + {%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %} [[inputs.exec]] commands = [ @@ -358,12 +382,6 @@ timeout = "15s" {%- endif %} -{%- if salt['pillar.get']('healthcheck:enabled', False) %} -[[inputs.file]] - files = ["/host/nsm/zeek/logs/zeek_restart.log"] - data_format = "influx" -{%- endif %} - [[inputs.file]] files = ["/etc/telegraf/node_config.json"] name_override = "node_config" diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index fe2625bbc..511cd0b89 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -18,7 +18,7 @@ {% if GLOBALS.pipeline != 'REDIS' %} {# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #} -{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %} +{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-heavynode'] %} {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %} {% endif %} {% endif %} diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 6b2f1333d..bc7afb609 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -10,7 +10,7 @@ # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l) + FILES=$(find /host/nsm/strelka/unprocessed -maxdepth 1 -type f | wc -l) echo "faffiles files=$FILES" fi diff --git a/salt/telegraf/scripts/containers.sh b/salt/telegraf/scripts/containers.sh index 836d4a999..3ba991f78 100644 --- a/salt/telegraf/scripts/containers.sh +++ b/salt/telegraf/scripts/containers.sh @@ -12,8 +12,9 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then CONTAINERSLOG=/var/log/somon/containers.log # the collector rewrites this every minute; report nothing rather than repeating a stale - # file as if it were current, in case a run was skipped or the collector is wedged - MAXAGE=150 + # file as if it were current, in case a run was skipped or the collector is wedged. + # Kept under two collector cycles so a 60s telegraf interval can't emit the same snapshot twice. + MAXAGE=90 if [ -r "$CONTAINERSLOG" ]; then AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") )) diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 10193cc38..fdaaa149b 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -11,27 +11,40 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then PREVCOUNTFILE='/tmp/eps.txt' - EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" + STATS="$(curl -s localhost:9600/_node/stats)" - if [ ! -z "$EVENTCOUNTCURRENT" ]; then - - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "consumptioneps eps=${EVENTS%%.*}" + if echo "$STATS" | jq -e '.pipelines.search' > /dev/null 2>&1; then + STAGE=search + COUNT="$(echo "$STATS" | jq '.pipelines.search.events.out')" + elif echo "$STATS" | jq -e '.pipelines.receiver' > /dev/null 2>&1; then + STAGE=receiver + COUNT="$(echo "$STATS" | jq '.pipelines.receiver.events.in')" + else + exit 0 fi + if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then + exit 0 + fi + + NOW=$(date +%s) + read -r PREVEPOCH PREVCOUNT 2>/dev/null < "$PREVCOUNTFILE" + echo "${NOW} ${COUNT}" > "$PREVCOUNTFILE" + + if ! [[ "$PREVEPOCH" =~ ^[0-9]+$ && "$PREVCOUNT" =~ ^[0-9]+$ ]]; then + exit 0 + fi + + ELAPSED=$((NOW - PREVEPOCH)) + DELTA=$((COUNT - PREVCOUNT)) + if [ "$ELAPSED" -le 0 ] || [ "$DELTA" -lt 0 ]; then + EPS=0 + else + EPS=$((DELTA / ELAPSED)) + fi + + echo "consumptioneps,stage=${STAGE} eps=${EPS}" + fi exit 0 diff --git a/salt/telegraf/scripts/features.sh b/salt/telegraf/scripts/features.sh index 7c4fe6f52..bcc47ecb1 100644 --- a/salt/telegraf/scripts/features.sh +++ b/salt/telegraf/scripts/features.sh @@ -10,8 +10,7 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled) LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled) - echo "features fps=$FPS_ENABLED" - echo "features lks=$LKS_ENABLED" + echo "features fps=$FPS_ENABLED,lks=$LKS_ENABLED" fi exit 0 From 4e0558d2d3b2f092b7d632073446ea9d151f7460 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:30:01 -0600 Subject: [PATCH 03/17] remove unused localrules config --- salt/docker/defaults.yaml | 1 - salt/firewall/defaults.yaml | 10 -- salt/firewall/soc_firewall.yaml | 3 - salt/manager/tools/sbin/so-yaml.py | 29 ++++ salt/manager/tools/sbin/so-yaml_test.py | 217 ++++++++++++++++++++++++ salt/manager/tools/sbin/soup | 20 +++ salt/nginx/enabled.sls | 1 - salt/nginx/etc/nginx.conf | 15 -- 8 files changed, 266 insertions(+), 30 deletions(-) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index b1e0d9162..2b3cd7605 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -103,7 +103,6 @@ docker: - 80:80 - 443:443 - 8443:8443 - - 7788:7788 - 7789:7789 custom_bind_mounts: [] extra_hosts: [] diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 5c1229787..74edd8dba 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -118,10 +118,6 @@ firewall: tcp: - 5601 udp: [] - localrules: - tcp: - - 7788 - udp: [] nginx: tcp: - 80 @@ -200,7 +196,6 @@ firewall: - postgres - elasticsearch_rest - elasticsearch_node - - localrules sensor: portgroups: - beats_5044 @@ -391,7 +386,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - - localrules - sensoroni fleet: portgroups: @@ -611,7 +605,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - - localrules - sensoroni fleet: portgroups: @@ -829,7 +822,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - - localrules - sensoroni fleet: portgroups: @@ -1058,7 +1050,6 @@ firewall: - elastic_agent_update - endgame - strelka_frontend - - localrules fleet: portgroups: - elasticsearch_rest @@ -1518,7 +1509,6 @@ firewall: - elasticsearch_rest - elasticsearch_node - elastic_agent_control - - localrules sensor: portgroups: - beats_5044 diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index a5181e50f..1e855c8af 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -138,9 +138,6 @@ firewall: kibana: tcp: *tcpsettings udp: *udpsettings - localrules: - tcp: *tcpsettings - udp: *udpsettings nginx: tcp: *tcpsettings udp: *udpsettings diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index d6465e97b..a4126bedf 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -20,6 +20,7 @@ def showUsage(args): print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr) print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) + print(' purgelistitem - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr) print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr) print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr) print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr) @@ -199,6 +200,33 @@ def removelistitem(args): return 0 +def purgeListItem(content, listItem): + if isinstance(content, dict): + for value in content.values(): + purgeListItem(value, listItem) + elif isinstance(content, list): + content[:] = [item for item in content if item != listItem] + for item in content: + purgeListItem(item, listItem) + + +def purgelistitem(args): + if len(args) != 3: + print('Missing filename, key arg, or list item to purge', file=sys.stderr) + showUsage(None) + return 1 + + filename = args[0] + key = args[1] + listItem = args[2] + + content = loadYaml(filename) + purgeListItem(getKeyValue(content, key), convertType(listItem)) + writeYaml(filename, content) + + return 0 + + def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject): pieces = key.split(".", 1) if len(pieces) > 1: @@ -385,6 +413,7 @@ def main(): "append": append, "appendlistobject": appendlistobject, "removelistitem": removelistitem, + "purgelistitem": purgelistitem, "replacelistobject": replacelistobject, "get": get, "remove": remove, diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7fa4cf8a7..72a82f44e 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -709,6 +709,223 @@ class TestRemoveListItem(unittest.TestCase): self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) +class TestPurgeListItem(unittest.TestCase): + + def test_purgelistitem_missing_arg(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd", "help"] + soyaml.purgelistitem(["file", "key"]) + sysmock.assert_called() + self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue()) + + def test_purgelistitem(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [a,b,c], key2: false}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- c\nkey2: false\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_nested_lists(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - c\n child2:\n deep1:\n - d\n deep2: 45\nkey2:\n- b\n- e\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_duplicates(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [b,a,b,c,b]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_list_of_objects(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- name: item1\n values:\n - a\n- name: item2\n values:\n - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_typed(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [1, '1', 2]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "1"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- '1'\n- 2\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_item_not_in_list(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c] }}") + file.close() + + soyaml.purgelistitem([filename, "key1", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - b\n - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_key_noexist(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [a,b,c]}") + file.close() + + result = soyaml.purgelistitem([filename, "key2.child1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- b\n- c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_key_noexist_deep(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c] }}") + file.close() + + result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - b\n - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_scalar_and_null_values(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: b, child2: null, child3: [b] }}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: b\n child2: null\n child3: []\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_lists_of_lists(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [[a,b], [b,[b,c]], b]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- - a\n- - - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_file_object(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}") + file.close() + + itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml" + file = open(itemfile, "w") + file.write("name: x\n") + file.close() + + soyaml.purgelistitem([filename, "key1", "file:" + itemfile]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- name: y\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_main(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c] }, key2: [b]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"] + soyaml.main() + sysmock.assert_called_once_with(0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - c\nkey2:\n- b\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_with_remove(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}") + file.close() + + soyaml.remove([filename, "key1.child1"]) + soyaml.purgelistitem([filename, "key2", "child1"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n" + self.assertEqual(actual, expected) + + class TestAppendListObject(unittest.TestCase): def test_appendlistobject_missing_arg(self): diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ba27ca004..89527d051 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -527,6 +527,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0 [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0 [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0 + [[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0 true } @@ -1202,6 +1203,25 @@ add_detection_runtime_fields() { } ### 3.4.0 End ### +### 3.5.0 Scripts ### +up_to_3.5.0() { + remove_localrules_portgroup + + INSTALLEDVERSION=3.5.0 +} + +remove_localrules_portgroup() { + local pillar_file + + for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do + grep -qs localrules "$pillar_file" || continue + echo "Removing the localrules portgroup from $pillar_file." + so-yaml.py remove "$pillar_file" firewall.portgroups.localrules + so-yaml.py purgelistitem "$pillar_file" firewall.role localrules + done +} +### 3.5.0 End ### + repo_sync() { echo "Sync the local repo." diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index f1a71238a..b25199ed9 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -64,7 +64,6 @@ so-nginx: - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro - /nsm/repo:/opt/socore/html/repo:ro - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro - - /nsm/rules:/nsm/rules:ro {% if NGINXMERGED.external_suricata %} - /opt/so/rules/nids/suri:/surirules:ro {% endif %} diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 5f9f9858d..e30b21adc 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -92,21 +92,6 @@ http { {%- endif %} {%- if GLOBALS.is_manager %} - - server { - listen 7788; - server_name {{ full_server_name }}; - root /nsm/rules; - location / { - allow all; - sendfile on; - sendfile_max_chunk 1m; - autoindex on; - autoindex_exact_size off; - autoindex_format html; - autoindex_localtime on; - } - } {%- if NGINXMERGED.external_suricata %} server { listen 7789 ssl; From 9bc3ae6b26bfbc6d0ea89b9ab6e9f00d96c667f5 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:49:34 -0600 Subject: [PATCH 04/17] further tighten elastic artifact registry --- salt/nginx/enabled.sls | 9 ++++++++- salt/nginx/etc/nginx.conf | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index f1a71238a..c6b20f7f5 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -16,6 +16,12 @@ include: - nginx.config - nginx.sostatus +fleet_beats_dir_nginx: + file.directory: + - name: /nsm/elastic-fleet/artifacts/beats + - makedirs: True + - show_changes: False + {% if GLOBALS.role != 'so-fleet' %} {% set container_config = 'so-nginx' %} make-rule-dir-nginx: @@ -54,7 +60,7 @@ so-nginx: - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - - /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro + - /nsm/elastic-fleet/artifacts/beats/:/opt/socore/elastic-artifacts/artifacts/beats:ro {% if GLOBALS.is_manager %} - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro @@ -106,6 +112,7 @@ so-nginx: {% endif %} - require: - file: nginxconf + - file: fleet_beats_dir_nginx {% if GLOBALS.is_manager %} {% if NGINXMERGED.ssl.replace_cert %} - file: managerssl_key diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 5f9f9858d..6a128d56b 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -57,7 +57,7 @@ http { server_name {{ GLOBALS.hostname }}; {%- endif %} root /opt/socore/elastic-artifacts; - location /artifacts/ { + location /artifacts/beats/ { try_files $uri =404; } location / { From 03317ba5e4860f21df3d7fe28b42e3a487629463 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 12 Nov 2026 20:42:29 -0500 Subject: [PATCH 05/17] Postgres: Telegraf partition maintenance over the socket, with grid-test fixes --- salt/manager/tools/sbin/soup | 36 +- salt/postgres/defaults.yaml | 1 + salt/postgres/soc_postgres.yaml | 6 + salt/postgres/tools/sbin/so-stats-show | 11 +- salt/postgres/tools/sbin/so-telegraf-postgres | 266 +++++++++++---- salt/postgres/tools/sbin/so-telegraf-repair | 308 +++++++++++++----- 6 files changed, 468 insertions(+), 160 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ba27ca004..165af7974 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -546,6 +546,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0 [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0 + [[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0 # All applicable post-upgrade steps completed; clear the resume marker. rm -f "$POSTVERSION_FILE" true @@ -1065,17 +1066,31 @@ telegraf_repair() { # when there is something to repair, so everyone else is left alone. local repair=/usr/sbin/so-telegraf-repair [[ -x "$repair" ]] || return 0 - docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0 + local state + state=$(docker inspect -f '{{.State.Running}} {{.State.Restarting}}' so-postgres 2>/dev/null) || return 0 + if [[ "$state" != "true false" ]]; then + echo " so-postgres is not running; skipping the Telegraf partition check." + FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres is not running, so Telegraf partitions were not checked. If /nsm is full, follow \"Telegraf Partition Repair\" at:") + FINAL_MESSAGE_QUEUE+=(" - https://securityonion.net/docs/postgresql") + return 0 + fi echo "Checking Telegraf metric partitions." - local status=0 - "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$? + local out status=0 + out=$("$repair" --check 2>&1) || status=$? + echo "$out" >> "$SOUP_LOG" case "$status" in 0) echo " Telegraf partitions are healthy; nothing to repair." ;; 1) echo " Repairing stalled Telegraf partitions." - "$repair" --yes \ - || echo " warning: so-telegraf-repair failed; run it manually" >&2 ;; - *) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;; + if ! "$repair" --yes; then + echo " warning: so-telegraf-repair failed; run it manually" >&2 + FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partition repair failed; Postgres metrics will keep filling /nsm until it is repaired:") + FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair") + fi ;; + 2) echo " Skipping: $(tail -n 1 <<<"$out")" ;; + *) echo " warning: could not check Telegraf partitions" >&2 + FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partitions could not be checked during the upgrade; run:") + FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair --check") ;; esac } @@ -1202,6 +1217,15 @@ add_detection_runtime_fields() { } ### 3.4.0 End ### +### 3.5.0 Scripts ### + +post_to_3.5.0() { + telegraf_repair + + set_postversion 3.5.0 +} +### 3.5.0 End ### + repo_sync() { echo "Sync the local repo." diff --git a/salt/postgres/defaults.yaml b/salt/postgres/defaults.yaml index 754a0d44c..12925b318 100644 --- a/salt/postgres/defaults.yaml +++ b/salt/postgres/defaults.yaml @@ -17,3 +17,4 @@ postgres: log_min_messages: 'warning' shared_preload_libraries: pg_cron cron.database_name: postgres + cron.host: /run/postgresql diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml index 4ab1e4dac..0c3af92f1 100644 --- a/salt/postgres/soc_postgres.yaml +++ b/salt/postgres/soc_postgres.yaml @@ -87,3 +87,9 @@ postgres: global: True advanced: True helpLink: postgres + cron.host: + description: Where pg_cron connects to run scheduled jobs such as Telegraf partition maintenance. Must be the container's PostgreSQL socket directory, /run/postgresql, because pg_cron connects without a password and only local socket connections are trusted. Any other value makes every scheduled job fail. Changing it restarts PostgreSQL. + readonly: True + global: True + advanced: True + helpLink: postgres diff --git a/salt/postgres/tools/sbin/so-stats-show b/salt/postgres/tools/sbin/so-stats-show index 3cf7a05d8..8fb180370 100644 --- a/salt/postgres/tools/sbin/so-stats-show +++ b/salt/postgres/tools/sbin/so-stats-show @@ -52,7 +52,7 @@ if [ -n "$FILTER_HOST" ] && ! [[ "$FILTER_HOST" =~ $HOST_RE ]]; then fi so_psql() { - docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@" + docker exec -e PGOPTIONS='-c application_name=so-stats-show' so-postgres psql -U so_maint_telegraf -d so_telegraf -At -F $'\t' "$@" } if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then @@ -60,6 +60,15 @@ if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | exit 2 fi +role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null) +if [ -z "$role_ok" ]; then + echo "The so_maint_telegraf role is missing; run a highstate." + exit 1 +elif [ "$role_ok" != t ]; then + echo "The so_maint_telegraf role is misconfigured; run a highstate." + exit 1 +fi + table_exists() { local table="$1" [ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ] diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index df7d16e65..7e907cc00 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -4,15 +4,18 @@ set -e # Provision Telegraf state inside the so-postgres container. # Usage: so-telegraf-postgres # create_db Ensure the so_telegraf database exists. -# group_role Provision the so_telegraf group role, telegraf/partman schemas, -# pg_partman, the so_admin maintenance routines, and the hourly -# pg_cron maintenance job. +# group_role Provision the so_telegraf group role, the so_maint_telegraf +# maintenance login role, telegraf/partman schemas, pg_partman, +# the so_admin maintenance routines, and the hourly pg_cron +# maintenance job. # user Create or update a per-minion login role granted to so_telegraf. # Env: ROLE_USER, ROLE_PASS. # retention Reconcile partman retention and premake on telegraf parents. # Env: RETENTION_DAYS. -# maintenance Drain default partitions and run partman maintenance. -# check Report partition health. Non-zero if any parent is unhealthy. +# maintenance Drain default partitions and run partman maintenance, as +# so_maint_telegraf. +# check Report partition health, as so_maint_telegraf. Non-zero if any +# parent is unhealthy. # # A default partition holding rows for a day blocks creating that day's child, # so maintenance drains defaults before calling partman. Use so-telegraf-repair @@ -20,6 +23,10 @@ set -e cmd="${1:?subcommand required}" +# Long enough for an hourly maintenance run to release its locks, short enough +# that soup and admins are not left hanging. +LOCK_TIMEOUT=60s + case "$cmd" in create_db) if ! docker exec so-postgres psql -U postgres -tAc \ @@ -30,7 +37,9 @@ case "$cmd" in ;; group_role) - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v lock_timeout="$LOCK_TIMEOUT" \ + -U postgres -d so_telegraf <<'EOSQL' +SET lock_timeout = :'lock_timeout'; DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN @@ -38,6 +47,17 @@ BEGIN END IF; END $$; +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_maint_telegraf') THEN + CREATE ROLE so_maint_telegraf; + END IF; +END +$$; +ALTER ROLE so_maint_telegraf LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD NULL; +GRANT so_telegraf TO so_maint_telegraf; +ALTER ROLE so_maint_telegraf SET role = so_telegraf; +ALTER ROLE so_maint_telegraf SET search_path = pg_catalog; GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf; CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf; GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf; @@ -58,52 +78,90 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman ALTER DEFAULT PRIVILEGES IN SCHEMA partman GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf; --- pg_cron runs these as postgres, so they must not sit in a schema any --- Telegraf role can create objects in. +-- Owned by postgres so no Telegraf role can create objects here; the routines +-- run with the caller's privileges. CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres; REVOKE ALL ON SCHEMA so_admin FROM PUBLIC; +-- Resolved through pg_inherits, never by name, so a view or a table that only +-- carries the _default name is never read or written. +CREATE OR REPLACE FUNCTION so_admin.telegraf_defaults() +RETURNS TABLE ( + parent_table regclass, + default_part regclass, + has_rows boolean, + default_size bigint +) +LANGUAGE plpgsql +STABLE +SECURITY INVOKER +AS $func$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT c.oid::regclass AS pt + FROM partman.part_config pc + JOIN pg_catalog.pg_class c ON c.oid = to_regclass(pc.parent_table) + WHERE pc.parent_table LIKE 'telegraf.%' + AND c.relkind = 'p' + ORDER BY pc.parent_table + LOOP + parent_table := r.pt; + + SELECT c.oid::regclass INTO default_part + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = r.pt + AND c.relkind = 'r' AND c.relispartition + AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'; + + IF default_part IS NULL THEN + has_rows := false; + default_size := NULL; + ELSE + EXECUTE format('SELECT EXISTS (SELECT 1 FROM %s LIMIT 1)', default_part) + INTO has_rows; + default_size := pg_catalog.pg_total_relation_size(default_part); + END IF; + + RETURN NEXT; + END LOOP; +END; +$func$; + CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance() LANGUAGE plpgsql +SECURITY INVOKER +-- No SET clause: it would forbid the COMMITs below, so so_maint_telegraf pins search_path instead. AS $proc$ DECLARE - r record; - v_default text; - v_rows bigint; + r record; BEGIN -- No per-parent EXCEPTION handler: partition_data_proc commits internally, -- and COMMIT is illegal while a subtransaction is active. A failing parent -- aborts the run and the next pass retries. FOR r IN - SELECT parent_table, retention - FROM partman.part_config - WHERE parent_table LIKE 'telegraf.%' - ORDER BY parent_table + SELECT pc.parent_table, pc.retention, d.default_part, d.default_size + FROM so_admin.telegraf_defaults() d + JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table + WHERE d.default_part IS NOT NULL AND d.has_rows + ORDER BY pc.parent_table LOOP - v_default := format('%I.%I', - split_part(r.parent_table, '.', 1), - split_part(r.parent_table, '.', 2) || '_default'); - - CONTINUE WHEN to_regclass(v_default) IS NULL; - - EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows; - CONTINUE WHEN v_rows = 0; - - RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining', - v_rows, v_default; + RAISE WARNING 'so_admin.telegraf_maintenance: % holds rows (%), draining', + r.default_part, pg_size_pretty(r.default_size); -- Cheaper to delete expired rows than to repartition and then drop them. IF r.retention IS NOT NULL THEN EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval', - v_default, r.retention); + r.default_part, r.retention); COMMIT; END IF; -- Bounded so a large backlog drains across several runs. CALL partman.partition_data_proc( p_parent_table := r.parent_table, - p_loop_count := 200, - p_source_table := v_default + p_loop_count := 200 ); COMMIT; END LOOP; @@ -112,58 +170,75 @@ BEGIN END; $proc$; -CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status() +DROP FUNCTION IF EXISTS so_admin.telegraf_partition_status(); +CREATE FUNCTION so_admin.telegraf_partition_status() RETURNS TABLE ( - parent_table text, - oldest_child date, - newest_child date, - days_ahead int, - retention text, - default_rows bigint, - default_size text + parent_table text, + oldest_child date, + newest_child date, + days_ahead int, + retention text, + default_has_rows boolean, + default_size text ) LANGUAGE plpgsql +STABLE AS $func$ DECLARE - r record; - v_default regclass; + r record; BEGIN FOR r IN - SELECT pc.parent_table AS pt, pc.retention AS ret - FROM partman.part_config pc - WHERE pc.parent_table LIKE 'telegraf.%' - ORDER BY pc.parent_table + SELECT d.parent_table AS pt, d.has_rows, d.default_size AS size, pc.retention AS ret + FROM so_admin.telegraf_defaults() d + JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table + ORDER BY d.parent_table::text LOOP - parent_table := r.pt; + parent_table := r.pt::text; retention := r.ret; - SELECT min(d), max(d) INTO oldest_child, newest_child + SELECT min(x), max(x) INTO oldest_child, newest_child FROM ( - SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d - FROM pg_inherits i - JOIN pg_class c ON c.oid = i.inhrelid - WHERE i.inhparent = r.pt::regclass - AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' + SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS x + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = r.pt + AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' ) s; - days_ahead := newest_child - current_date; - - v_default := to_regclass(format('%I.%I', - split_part(r.pt, '.', 1), - split_part(r.pt, '.', 2) || '_default')); - IF v_default IS NULL THEN - default_rows := NULL; - default_size := NULL; - ELSE - EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows; - default_size := pg_size_pretty(pg_total_relation_size(v_default)); - END IF; + days_ahead := newest_child - current_date; + default_has_rows := r.has_rows; + default_size := pg_size_pretty(r.size); RETURN NEXT; END LOOP; END; $func$; +GRANT USAGE ON SCHEMA so_admin TO so_telegraf; +GRANT EXECUTE ON FUNCTION so_admin.telegraf_defaults() TO so_telegraf; +GRANT EXECUTE ON PROCEDURE so_admin.telegraf_maintenance() TO so_telegraf; +GRANT EXECUTE ON FUNCTION so_admin.telegraf_partition_status() TO so_telegraf; + +-- Partitions created by a superuser session (an older repair) belong to +-- postgres; partman acting as so_telegraf could not drop them. +DO $$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT n.nspname, c.relname + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND (n.nspname = 'telegraf' + OR (n.nspname = 'partman' AND c.relname LIKE 'template\_telegraf\_%')) + AND c.relowner <> 'so_telegraf'::regrole + LOOP + EXECUTE format('ALTER TABLE %I.%I OWNER TO so_telegraf', r.nspname, r.relname); + END LOOP; +END +$$; + -- Drop the registration older releases left in so_telegraf. SELECT CASE WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database() @@ -175,20 +250,69 @@ DROP EXTENSION pg_cron CASCADE; \endif EOSQL + stalled=false + probe_failed=false + if out=$(docker exec -e PGOPTIONS="-c lock_timeout=$LOCK_TIMEOUT" so-postgres \ + psql -U so_maint_telegraf -d so_telegraf -tAc \ + "SELECT coalesce(bool_or(has_rows), false) FROM so_admin.telegraf_defaults()"); then + case "$out" in + t) stalled=true ;; + f) stalled=false ;; + *) probe_failed=true ;; + esac + else + probe_failed=true + fi + if $probe_failed; then + echo "ERROR: could not check Telegraf default partitions; the maintenance job was not changed." >&2 + exit 1 + fi + # Guarded on the live GUC so applying this before the postgresql.conf change # has restarted the container skips rather than failing. - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL' + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v stalled="$stalled" \ + -U postgres -d postgres <<'EOSQL' SELECT CASE WHEN current_setting('cron.database_name', true) = current_database() THEN 'true' ELSE 'false' END AS cron_here \gset \if :cron_here CREATE EXTENSION IF NOT EXISTS pg_cron; --- cron.schedule_in_database is idempotent by jobname. +-- A highstate reconciles the job's definition but never its active flag; only so-telegraf-repair turns it on. +BEGIN; +SELECT coalesce(max(jobid)::text, '') AS jobid, + count(*) > 0 AS job_exists, + coalesce(bool_and(active), true) AS job_active +FROM cron.job +WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset +SELECT cron.unschedule(jobid) FROM cron.job +WHERE jobname = 'telegraf-partman-maintenance' AND username <> 'so_maint_telegraf'; +\if :job_exists +SELECT cron.alter_job(:jobid, + schedule := '17 * * * *', + command := 'CALL so_admin.telegraf_maintenance()', + database := 'so_telegraf'); +\else +-- Registered inactive on a stalled grid so nothing drains the default +-- partitions before so-telegraf-repair truncates them. SELECT cron.schedule_in_database( 'telegraf-partman-maintenance', '17 * * * *', 'CALL so_admin.telegraf_maintenance()', - 'so_telegraf' + 'so_telegraf', + 'so_maint_telegraf', + NOT :'stalled'::boolean ); +\endif +UPDATE cron.job SET nodename = current_setting('cron.host') +WHERE jobname = 'telegraf-partman-maintenance' + AND nodename IS DISTINCT FROM current_setting('cron.host'); +COMMIT; +SELECT coalesce(bool_and(active), true) AS job_active +FROM cron.job +WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset +\if :job_active +\else +\echo 'WARNING: the Telegraf maintenance job is inactive; run ''sudo so-telegraf-repair''.' +\endif \else \echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.' \endif @@ -241,13 +365,15 @@ SET retention = :'retention_days' || ' days', retention_keep_table = false, premake = 7, infinite_time_partitions = true -WHERE parent_table LIKE 'telegraf.%'; +WHERE parent_table LIKE 'telegraf.%' + AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c + WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p'); \endif EOSQL ;; maintenance) - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U so_maint_telegraf -d so_telegraf <<'EOSQL' SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL THEN 'true' ELSE 'false' END AS has_proc \gset \if :has_proc @@ -259,7 +385,7 @@ EOSQL ;; check) - docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL' + docker exec -i so-postgres psql -U so_maint_telegraf -d so_telegraf <<'EOSQL' \pset border 2 SELECT * FROM so_admin.telegraf_partition_status(); EOSQL @@ -278,9 +404,9 @@ LIMIT 5; \echo 'pg_cron is not installed in this database.' \endif EOSQL - unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \ + unhealthy=$(docker exec so-postgres psql -U so_maint_telegraf -d so_telegraf -tAc \ "SELECT count(*) FROM so_admin.telegraf_partition_status() - WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1") + WHERE default_has_rows OR coalesce(days_ahead, -1) < 1") if [ "${unhealthy:-1}" != "0" ]; then echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2 exit 1 diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair index 75004402e..f6d9cf9f9 100644 --- a/salt/postgres/tools/sbin/so-telegraf-repair +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -11,26 +11,38 @@ # maintenance once. Healthy grids are reported and left alone. # # Usage: so-telegraf-repair [--check] [--yes] [--no-restart] -# --check Report health and change nothing. +# --check Report health and change nothing. Also reports a missing +# so_maint_telegraf role and a maintenance job that is missing +# or inactive. # --yes Skip the confirmation prompt (for soup and other automation). # --no-restart Never restart so-postgres, even if pg_cron's launcher is dead. # +# Repair mode (without --check) registers the so_maint_telegraf role and the +# maintenance job if they are missing, pauses the job while it repairs, and +# always leaves it active afterwards. Data work runs as so_maint_telegraf. +# # Exit status: # 0 healthy, or repair completed -# 1 repair is needed (--check only) -# 2 cannot run here: so-postgres, so_telegraf or pg_partman is missing +# 1 repair is needed (--check), or the repair failed after it started +# 2 skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing +# 3 could not check, for example a connection error -set -e +set -eE # Matches p_premake in telegraf.conf's create_parent template. PREMAKE=7 JOB_NAME=telegraf-partman-maintenance +MAINT_ROLE=so_maint_telegraf +# Long enough for an hourly maintenance run to release its locks, short enough +# that soup and admins are not left hanging. +LOCK_TIMEOUT=60s +PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair" CHECK_ONLY=false ASSUME_YES=false NO_RESTART=false -usage() { sed -n '/^# Usage:/,/^# 2 /p' "$0" | sed 's/^# \?//'; } +usage() { sed -n '/^# Usage:/,/^# 3 /p' "$0" | sed 's/^# \?//'; } while [[ $# -gt 0 ]]; do case "$1" in @@ -38,75 +50,170 @@ while [[ $# -gt 0 ]]; do --yes|-y) ASSUME_YES=true ;; --no-restart) NO_RESTART=true ;; -h|--help) usage; exit 0 ;; - *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;; esac shift done -skip() { echo "$*"; exit 2; } +# Strips C0 (except tab and newline), DEL and C1 characters, so names and +# messages written with Telegraf credentials can't drive the terminal. +sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; } -psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } -psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; } +ERR_FILE=$(mktemp) +ERR_STATUS=3 +ERR_PREFIX="Could not check Telegraf partitions" +job_id= +restore_active= + +on_err() { + # Command substitutions inherit this trap; only the top level reports. + [[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS" + local line + line=$(tail -n 1 "$ERR_FILE" | sanitise) + echo "$ERR_PREFIX: ${line:-command failed}" >&2 + exit "$ERR_STATUS" +} + +on_exit() { + if [[ -n "$restore_active" ]]; then + psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true + fi + rm -f "$ERR_FILE" +} + +trap on_err ERR +trap on_exit EXIT + +finish() { trap - ERR; exit "$1"; } +skip() { echo "$*"; finish 2; } + +# Keeps the last stderr line for on_err while still showing all of it. +capture() { + local rc=0 + : >"$ERR_FILE" + "$@" 2>"$ERR_FILE" || rc=$? + sanitise <"$ERR_FILE" >&2 + return "$rc" +} + +psql_tg() { + capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@" +} +psql_pg() { + capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@" +} +psql_cron() { + capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@" +} + +# A part_config row counts only while its parent is a partitioned table. +TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%' + AND EXISTS (SELECT 1 FROM pg_class c + WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')" -# query_to_xml so the per-table row counts need no helper function installed. REPORT=" -WITH parents AS ( - SELECT pc.parent_table, - pc.premake, - split_part(pc.parent_table, '.', 1) AS sch, - split_part(pc.parent_table, '.', 2) AS tbl - FROM partman.part_config pc - WHERE pc.parent_table LIKE 'telegraf.%' -), children AS ( - SELECT p.parent_table, - max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child - FROM parents p - JOIN pg_class pt ON pt.oid = p.parent_table::regclass - JOIN pg_inherits i ON i.inhparent = pt.oid - JOIN pg_class c ON c.oid = i.inhrelid - WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' - GROUP BY p.parent_table -), defaults AS ( - SELECT p.parent_table, - p.premake, - format('%I.%I', p.sch, p.tbl || '_default') AS default_table, - to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid - FROM parents p -) SELECT d.parent_table, c.newest_child, (c.newest_child - current_date) AS days_ahead, - d.premake, - CASE WHEN d.default_oid IS NULL THEN NULL ELSE - (xpath('/row/cnt/text()', - query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table), - false, true, '')))[1]::text::bigint - END AS default_rows, - CASE WHEN d.default_oid IS NULL THEN NULL - ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size -FROM defaults d -LEFT JOIN children c ON c.parent_table = d.parent_table -ORDER BY 1 + pc.premake, + d.has_rows AS default_has_rows, + pg_size_pretty(d.default_size) AS default_size +FROM so_admin.telegraf_defaults() d +JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table +JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p' +LEFT JOIN LATERAL ( + SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child + FROM pg_inherits i + JOIN pg_class ch ON ch.oid = i.inhrelid + WHERE i.inhparent = d.parent_table + AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT' +) c ON true +ORDER BY d.parent_table::text " -docker ps --format '{{.Names}}' | grep -qx so-postgres \ +print_report() { + local report + report=$(psql_tg -c "$REPORT") + sanitise <<<"$report" +} + +has_cron() { + [[ -n "$cron_db" ]] || return 0 + psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'" +} + +lookup_job() { + local cron + cron=$(has_cron) + [[ -n "$cron" ]] || return 0 + psql_cron -tA -F '|' -c \ + "SELECT jobid, active FROM cron.job + WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE' + ORDER BY jobid LIMIT 1" +} + +output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only) +output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]') +output=${output:-INFLUXDB} +case "$output" in + POSTGRES|BOTH) ;; + *) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;; +esac + +containers=$(capture docker ps --format '{{.Names}}') +grep -qx so-postgres <<<"$containers" \ || skip "so-postgres is not running; nothing to repair." -docker exec so-postgres psql -U postgres -tAc \ - "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \ +has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'") +[[ -n "$has_db" ]] \ || skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres." -psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \ + +# so_telegraf before the postgres state lands, postgres after. +cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)") +cron_db=$(tr -d '[:space:]' <<<"$cron_db") + +role_ok=$(psql_pg -tAc \ + "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) + FROM pg_roles WHERE rolname = '$MAINT_ROLE'") +job= +[[ "$cron_db" == postgres ]] && job=$(lookup_job) + +if [[ "$role_ok" != t ]] && $CHECK_ONLY; then + echo "Repair is needed:" + if [[ -z "$role_ok" ]]; then + echo " * maintenance role $MAINT_ROLE is missing" + else + echo " * maintenance role $MAINT_ROLE is misconfigured" + fi + echo + echo "Re-run without --check to repair." + finish 1 +fi + +if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then + echo "Registering the maintenance role and job." + if ! /usr/sbin/so-telegraf-postgres group_role; then + echo "Could not register the maintenance job; run a highstate and try again." >&2 + finish 3 + fi + [[ "$cron_db" == postgres ]] && job=$(lookup_job) +fi + +has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'") +[[ -n "$has_partman" ]] \ || skip "pg_partman is not installed in so_telegraf; nothing to repair." parents=$(psql_tg -tAc \ - "SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'") -stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t") -behind=$(psql_tg -tAc \ - "SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1") + "SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS") +counts=$(psql_tg -tAc \ + "SELECT count(*) FILTER (WHERE default_has_rows) || '|' || + count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1) + FROM ( $REPORT ) t") +IFS='|' read -r stranded behind <<<"$counts" # premake < 7, or infinite_time_partitions off: without the latter partman # refuses to premake forward across the gap the stall left behind. misconfigured=$(psql_tg -tAc \ "SELECT count(*) FROM partman.part_config - WHERE parent_table LIKE 'telegraf.%' + WHERE $TELEGRAF_PARENTS AND (premake < $PREMAKE OR NOT infinite_time_partitions)") # Both columns are matched because which one carries the launcher's name varies @@ -115,16 +222,24 @@ launcher=$(psql_pg -tAc \ "SELECT count(*) FROM pg_stat_activity WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'") -# so_telegraf before the postgres state lands, postgres after. -cron_db=$(docker exec so-postgres psql -U postgres -tAc \ - "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') last_run=never -if [[ -n "$cron_db" ]]; then - last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ - "SELECT coalesce(max(d.start_time)::text, 'never') +cron_installed=$(has_cron) +if [[ -n "$cron_installed" ]]; then + run=$(psql_cron -tAc \ + "SELECT d.status || ' at ' || d.start_time FROM cron.job_run_details d JOIN cron.job j USING (jobid) - WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown) - [[ -n "$last_run" ]] || last_run=never + WHERE j.jobname = '$JOB_NAME' + ORDER BY d.start_time DESC LIMIT 1") + if [[ -n "$run" ]]; then + message=$(psql_cron -tAc \ + "SELECT coalesce(d.return_message, '') + FROM cron.job_run_details d JOIN cron.job j USING (jobid) + WHERE j.jobname = '$JOB_NAME' + ORDER BY d.start_time DESC LIMIT 1") + message=$(sanitise <<<"$message") + message=${message//$'\n'/ } + last_run="$(sanitise <<<"$run"): ${message:0:200}" + fi fi # A grid that has never written a metric has nothing to recover, and an empty @@ -132,43 +247,67 @@ fi restart_needed=false [[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true +# group_role registers the job only in the postgres database. +job_inactive=false +if [[ "$cron_db" == postgres ]]; then + [[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true +fi + repair_needed=false [[ "$stranded" -gt 0 ]] && repair_needed=true [[ "$behind" -gt 0 ]] && repair_needed=true [[ "$misconfigured" -gt 0 ]] && repair_needed=true $restart_needed && repair_needed=true +$job_inactive && repair_needed=true echo "Telegraf partition status:" -psql_tg -c "$REPORT" -echo "Rows stranded in default partitions: $stranded" -echo "pg_cron metadata database: ${cron_db:-unset}" +print_report +echo "Default partitions with rows: $stranded" +echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")" +[[ "$cron_db" == postgres ]] \ + || echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job." echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)" echo "Last $JOB_NAME run: $last_run" echo if ! $repair_needed; then echo "Telegraf partitions are healthy. Nothing to do." - exit 0 + finish 0 fi if $CHECK_ONLY; then echo "Repair is needed:" - [[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions" + [[ "$stranded" -gt 0 ]] && echo " * $stranded default partition(s) hold rows" [[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window" [[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings" $restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all" + $job_inactive && echo " * the maintenance job is missing or inactive" echo echo "Re-run without --check to repair." - exit 1 + finish 1 fi if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then - echo "This will permanently discard the $stranded stranded row(s) above." + echo "This will permanently discard the rows in the $stranded default partition(s) above." $restart_needed && ! $NO_RESTART && \ echo "so-postgres will also be restarted, which briefly interrupts SOC." - [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; } + [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; } read -r -p "Continue? [y/N] " answer - [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } + [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; } +fi + +ERR_STATUS=1 +ERR_PREFIX="Telegraf partition repair failed" + +if [[ "$cron_db" == postgres ]]; then + if [[ -z "$job" ]]; then + echo "Could not register the maintenance job; run a highstate and try again." >&2 + finish 3 + fi + job_id=${job%%|*} + [[ "${job##*|}" == t ]] && restore_active=true || restore_active=false + echo "Pausing the $JOB_NAME job during the repair." + psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null fi if [[ "$misconfigured" -gt 0 ]]; then @@ -178,7 +317,7 @@ if [[ "$misconfigured" -gt 0 ]]; then "UPDATE partman.part_config SET premake = GREATEST(premake, $PREMAKE), infinite_time_partitions = true - WHERE parent_table LIKE 'telegraf.%'" + WHERE $TELEGRAF_PARENTS" fi if [[ "$stranded" -gt 0 ]]; then @@ -186,27 +325,24 @@ if [[ "$stranded" -gt 0 ]]; then # One transaction: Telegraf is still writing, so a default emptied without # its partition in place is refilled before maintenance can attach one. psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' +BEGIN; DO $$ DECLARE r record; BEGIN FOR r IN - SELECT pc.parent_table, - format('%I.%I', n.nspname, c.relname) AS default_table - FROM partman.part_config pc - JOIN pg_class p ON p.oid = pc.parent_table::regclass - JOIN pg_inherits i ON i.inhparent = p.oid - JOIN pg_class c ON c.oid = i.inhrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' + SELECT pc.parent_table, d.default_part + FROM so_admin.telegraf_defaults() d + JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table + WHERE d.default_part IS NOT NULL LOOP - EXECUTE format('TRUNCATE TABLE %s', r.default_table); + EXECUTE format('TRUNCATE TABLE %s', r.default_part); PERFORM partman.create_partition_time( r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]); END LOOP; END $$; +COMMIT; EOSQL fi @@ -222,7 +358,7 @@ if $restart_needed; then sleep 2 done docker exec so-postgres pg_isready -U postgres -q \ - || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; } + || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; } fi fi @@ -238,8 +374,14 @@ CALL partman.run_maintenance_proc(); \endif EOSQL +if [[ -n "$job_id" ]]; then + psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null + restore_active= + echo "The $JOB_NAME job is active." +fi + echo echo "Telegraf partition status after repair:" -psql_tg -c "$REPORT" +print_report echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:" echo " so-telegraf-repair --check" From b01f01da82be0dd7da6e8a48c35fb5728b51c85d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 13 Nov 2026 16:07:55 -0500 Subject: [PATCH 06/17] Fix formatting issues in soup script --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 165af7974..774cc322c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -17,7 +17,9 @@ INSTALLEDVERSION=$(cat /etc/soversion) # not finish its post-upgrade steps; its contents are the resume point. It is read # here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate # stamps /etc/soversion from the pillar. + POSTVERSION_FILE=/etc/sopostversion + if [ -f "$POSTVERSION_FILE" ]; then POSTVERSION=$(cat "$POSTVERSION_FILE") else From 01518ec25ec9f4cd32e280a00d79c85bf7d6eb0e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 16 Nov 2026 11:14:06 -0500 Subject: [PATCH 07/17] Fix formatting issues in soup script --- salt/manager/tools/sbin/soup | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 774cc322c..1be2783a0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -17,7 +17,6 @@ INSTALLEDVERSION=$(cat /etc/soversion) # not finish its post-upgrade steps; its contents are the resume point. It is read # here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate # stamps /etc/soversion from the pillar. - POSTVERSION_FILE=/etc/sopostversion if [ -f "$POSTVERSION_FILE" ]; then From f492e87f20c1e23e55a16b39eb3b3fd15c64c9a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 16 Nov 2026 12:12:11 -0500 Subject: [PATCH 08/17] Add check for post-upgrade version file --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1be2783a0..774cc322c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -17,6 +17,7 @@ INSTALLEDVERSION=$(cat /etc/soversion) # not finish its post-upgrade steps; its contents are the resume point. It is read # here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate # stamps /etc/soversion from the pillar. + POSTVERSION_FILE=/etc/sopostversion if [ -f "$POSTVERSION_FILE" ]; then From 52f9fe3092139d28db673bea72395abfe6e61d68 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 16 Nov 2026 13:52:14 -0500 Subject: [PATCH 09/17] Remove blank line in soup script --- salt/manager/tools/sbin/soup | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 774cc322c..f0e250b56 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -5,7 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - . /usr/sbin/so-common . /usr/sbin/so-image-common From 5558c546328d317f2abaf91fa4378969d737f0f0 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:44:58 -0600 Subject: [PATCH 10/17] so-yaml: treat non-mapping parents as missing keys getKeyValue and removeKey now return early when a parent of the key is null or not a mapping, so get, remove and purgelistitem report the key as missing instead of crashing on a hand-edited pillar such as `firewall: null`. --- salt/manager/tools/sbin/so-yaml.py | 4 ++ salt/manager/tools/sbin/so-yaml_test.py | 61 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index a4126bedf..c38d523db 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -319,6 +319,8 @@ def add(args): def removeKey(content, key): + if not isinstance(content, dict): + return pieces = key.split(".", 1) if len(pieces) > 1: if pieces[0] in content: @@ -362,6 +364,8 @@ def replace(args): def getKeyValue(content, key): + if not isinstance(content, dict): + return None pieces = key.split(".", 1) if len(pieces) > 1 and pieces[0] in content: return getKeyValue(content[pieces[0]], pieces[1]) diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 72a82f44e..71f748703 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -128,6 +128,22 @@ class TestRemove(unittest.TestCase): sysmock.assert_called_once_with(1) self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue()) + def test_remove_null_parent(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("key1: null\n") + file.close() + + result = soyaml.remove([filename, "key1.child1.deep1"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1: null\n" + self.assertEqual(actual, expected) + def test_append_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: with patch('sys.stderr', new=StringIO()) as mock_stderr: @@ -565,6 +581,19 @@ class TestRemove(unittest.TestCase): self.assertEqual(result, 2) self.assertEqual("", mock_stdout.getvalue()) + def test_get_null_parent(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("key1: null\n") + file.close() + + result = soyaml.get([filename, "key1.child1"]) + self.assertEqual(result, 2) + self.assertEqual("", mock_stdout.getvalue()) + self.assertIn("Key 'key1.child1' not found by so-yaml.py", mock_stderr.getvalue()) + def test_get_empty_file(self): with patch('sys.stdout', new=StringIO()) as mock_stdout: with patch('sys.stderr', new=StringIO()) as mock_stderr: @@ -925,6 +954,38 @@ class TestPurgeListItem(unittest.TestCase): expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n" self.assertEqual(actual, expected) + def test_purgelistitem_null_parent(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("key1: null\n") + file.close() + + result = soyaml.purgelistitem([filename, "key1.child1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1: null\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_non_dict_parent(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("key1:\n- a\n- b\n") + file.close() + + result = soyaml.purgelistitem([filename, "key1.child1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- b\n" + self.assertEqual(actual, expected) + class TestAppendListObject(unittest.TestCase): From 8c3611ac72765fe53929bada9f7dc66327fa54fd Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:46:10 -0600 Subject: [PATCH 11/17] only rewrite pillar if it has `localrules` reference(s) --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 89527d051..6bd55788a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1215,6 +1215,7 @@ remove_localrules_portgroup() { for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do grep -qs localrules "$pillar_file" || continue + so-yaml.py get "$pillar_file" firewall 2>/dev/null | grep -q '\' || continue echo "Removing the localrules portgroup from $pillar_file." so-yaml.py remove "$pillar_file" firewall.portgroups.localrules so-yaml.py purgelistitem "$pillar_file" firewall.role localrules From e7bb00d19def120bd6d542f00ae02df36f1b991a Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:47:25 -0600 Subject: [PATCH 12/17] move /nsm/rules ownership into soc/config.sls. nginx no longer serves /nsm/rules, but so-soc still mounts it read-write --- salt/nginx/enabled.sls | 23 ++++++----------------- salt/soc/config.sls | 10 ++++++++++ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index b25199ed9..8a815ac45 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -9,6 +9,12 @@ {% from 'ca/trust.map.jinja' import CATRUST %} {% from 'docker/docker.map.jinja' import DOCKERMERGED %} {% from 'nginx/map.jinja' import NGINXMERGED %} +{% if GLOBALS.role != 'so-fleet' %} +{% set container_config = 'so-nginx' %} +{% else %} +{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #} +{% set container_config = 'so-nginx-fleet-node' %} +{% endif %} include: - ca.trust @@ -16,23 +22,6 @@ include: - nginx.config - nginx.sostatus -{% if GLOBALS.role != 'so-fleet' %} -{% set container_config = 'so-nginx' %} -make-rule-dir-nginx: - file.directory: - - name: /nsm/rules - - user: socore - - group: socore - - recurse: - - user - - group - - show_changes: False - -{% else %} -{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #} -{% set container_config = 'so-nginx-fleet-node' %} -{% endif %} - so-nginx: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }} diff --git a/salt/soc/config.sls b/salt/soc/config.sls index be2887897..53ee7f40a 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -251,6 +251,16 @@ socsensoronirepos: - mode: 775 - makedirs: True +nsm_rules_dir: + file.directory: + - name: /nsm/rules + - user: socore + - group: socore + - recurse: + - user + - group + - show_changes: False + create_custom_local_yara_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-yara From cac027d4384aacc4775f35ff0cc9d60dfbddc5ef Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:52:05 -0600 Subject: [PATCH 13/17] fix merge --- salt/nginx/enabled.sls | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index 94318b835..f508d6c0a 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -10,18 +10,6 @@ {% from 'docker/docker.map.jinja' import DOCKERMERGED %} {% from 'nginx/map.jinja' import NGINXMERGED %} -include: - - ca.trust - - nginx.ssl - - nginx.config - - nginx.sostatus - -fleet_beats_dir_nginx: - file.directory: - - name: /nsm/elastic-fleet/artifacts/beats - - makedirs: True - - show_changes: False - {% if GLOBALS.role != 'so-fleet' %} {% set container_config = 'so-nginx' %} {% else %} @@ -35,6 +23,12 @@ include: - nginx.config - nginx.sostatus +fleet_beats_dir_nginx: + file.directory: + - name: /nsm/elastic-fleet/artifacts/beats + - makedirs: True + - show_changes: False + so-nginx: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }} From 0f5170e17eca358c5cfd382902108ade35536c41 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:19:22 -0600 Subject: [PATCH 14/17] soup missing closing } post merge resolution; --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 21d8619a8..10f37e537 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1236,6 +1236,7 @@ remove_localrules_portgroup() { so-yaml.py remove "$pillar_file" firewall.portgroups.localrules so-yaml.py purgelistitem "$pillar_file" firewall.role localrules done +} post_to_3.5.0() { telegraf_repair From 6d7be47a69ad58fd2e533011dfa6d432368ddf0b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 17 Nov 2026 14:17:27 -0500 Subject: [PATCH 15/17] Trim Telegraf collection on Postgres-only grids and add Postgres support to Elasticsearch growth scripts --- salt/common/tools/sbin/so-common | 102 ++++++++++ .../sbin/so-elasticsearch-indices-growth | 188 ++++++++++++------ .../sbin/so-elasticsearch-retention-estimate | 43 +++- salt/telegraf/etc/telegraf.conf | 31 +++ 4 files changed, 295 insertions(+), 69 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c8ebae084..929103de5 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -656,6 +656,108 @@ set_version() { fi } +so_telegraf_output() { + local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml" + local output + + output=$(lookup_pillar output telegraf) || return 1 + if [ -z "$output" ]; then + if [ ! -f "$defaultsfile" ]; then + echo "Could not find $defaultsfile" >&2 + return 1 + fi + output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]') + fi + echo "${output^^}" +} + +so_telegraf_pg_check() { + local app="$1" + local databases role_ok + + if ! docker_check_running postgres; then + echo "so-postgres is not running. Run $app on the manager." >&2 + return 1 + fi + if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then + echo "Unable to list the so-postgres databases." >&2 + return 1 + fi + if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then + echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2 + return 1 + fi + if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then + echo "Unable to query the so-postgres roles." >&2 + return 1 + fi + if [ -z "$role_ok" ]; then + echo "The so_maint_telegraf role is missing; run a highstate." >&2 + return 1 + elif [ "$role_ok" != t ]; then + echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2 + return 1 + fi +} + +so_telegraf_psql() { + local app="$1" + local sql="$2" + + docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \ + so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql" +} + +# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is +# each host's newest size in the past 4 hours; then is that host's newest size in the window +# [now - start, now - stop), or its current size when it has none there, so a host missing from the +# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits +# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a +# value can only come from a forged Telegraf writer. +so_es_index_size_growth() { + local app="$1" + local start="$2" + local stop="$3" + local relkind + + relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1 + [[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0 + + so_telegraf_psql "$app" "$(cat <= now() - interval '${start}' + AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$' + AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0 + AND f.value ~ '^[0-9]{1,15}$' +), +cur AS ( + SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes + FROM sizes + WHERE time >= now() - interval '4 hours' + ORDER BY tag_id, stream, time DESC +), +past AS ( + SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes + FROM sizes + WHERE time < now() - interval '${stop}' + ORDER BY tag_id, stream, time DESC +) +SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes)) +FROM cur c +LEFT JOIN past p USING (tag_id, stream) +WHERE EXISTS ( + SELECT 1 FROM telegraf.elasticsearch_index_size + WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}' +) +GROUP BY c.stream +ORDER BY c.stream +EOF +)" +} + status () { printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n" } diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth index 3381947eb..9acdaafcd 100644 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth @@ -5,6 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data +# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to +# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise. + INFLUX_URL="https://localhost:8086/api/v2" . /usr/sbin/so-common @@ -18,8 +22,6 @@ lookup_org_id() { echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id" } -ORG_ID=$(lookup_org_id) - run_flux_query() { local query=$1 request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null @@ -39,75 +41,137 @@ bytes_to_gb() { fi } -indexes_query='from(bucket: "telegraf/so_long_term") -|> range(start: -7d) -|> filter(fn: (r) => r._measurement == "elasticsearch_index_size") -|> distinct(column: "_field") -|> keep(columns: ["_field"])' +influxdb_indices_growth() { + ORG_ID=$(lookup_org_id) -indexes_result=$(run_flux_query "$indexes_query") -indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) + indexes_query='from(bucket: "telegraf/so_long_term") + |> range(start: -7d) + |> filter(fn: (r) => r._measurement == "elasticsearch_index_size") + |> distinct(column: "_field") + |> keep(columns: ["_field"])' -printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" -printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + indexes_result=$(run_flux_query "$indexes_query") + indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) -for index in $indexes; do - [[ -z "$index" ]] && continue - current_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -4h) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - current_result=$(run_flux_query "$current_query") - current_size=$(read_csv_result "$current_result") - current_size=${current_size:-0} + printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" + printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" - size_24h_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -25h, stop: -23h) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_24h_result=$(run_flux_query "$size_24h_query") - size_24h_ago=$(read_csv_result "$size_24h_result") - size_24h_ago=${size_24h_ago:-$current_size} + for index in $indexes; do + [[ -z "$index" ]] && continue + current_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -4h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + current_result=$(run_flux_query "$current_query") + current_size=$(read_csv_result "$current_result") + current_size=${current_size:-0} - size_7d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -7d8h, stop: -7d) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_7d_result=$(run_flux_query "$size_7d_query") - size_7d_ago=$(read_csv_result "$size_7d_result") - size_7d_ago=${size_7d_ago:-$current_size} + size_24h_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -25h, stop: -23h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_24h_result=$(run_flux_query "$size_24h_query") + size_24h_ago=$(read_csv_result "$size_24h_result") + size_24h_ago=${size_24h_ago:-$current_size} - size_30d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -30d8h, stop: -30d) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_30d_result=$(run_flux_query "$size_30d_query") - size_30d_ago=$(read_csv_result "$size_30d_result") - size_30d_ago=${size_30d_ago:-$current_size} + size_7d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -7d8h, stop: -7d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_7d_result=$(run_flux_query "$size_7d_query") + size_7d_ago=$(read_csv_result "$size_7d_result") + size_7d_ago=${size_7d_ago:-$current_size} - # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. - growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) + size_30d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -30d8h, stop: -30d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_30d_result=$(run_flux_query "$size_30d_query") + size_30d_ago=$(read_csv_result "$size_30d_result") + size_30d_ago=${size_30d_ago:-$current_size} - growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) + # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. + growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) - growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) + growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) - growth_24h_gb=$(bytes_to_gb "$growth_24h") - growth_7d_gb=$(bytes_to_gb "$growth_7d") - growth_30d_gb=$(bytes_to_gb "$growth_30d") + growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) - # Only results for indices with atleast 1 metric above 0.00 - if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then - printf "%020.2f|%-50s %15s %15s %15s\n" \ - "$growth_24h" \ - "$index" \ - "$growth_24h_gb" \ - "$growth_7d_gb" \ - "$growth_30d_gb" - fi -done | sort -t'|' -k1,1nr | cut -d'|' -f2- + growth_24h_gb=$(bytes_to_gb "$growth_24h") + growth_7d_gb=$(bytes_to_gb "$growth_7d") + growth_30d_gb=$(bytes_to_gb "$growth_30d") + # Only results for indices with atleast 1 metric above 0.00 + if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" \ + "$growth_24h" \ + "$index" \ + "$growth_24h_gb" \ + "$growth_7d_gb" \ + "$growth_30d_gb" + fi + done | sort -t'|' -k1,1nr | cut -d'|' -f2- +} + +postgres_indices_growth() { + local app="so-elasticsearch-indices-growth" + local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours") + local -a stops=("23 hours" "7 days" "30 days") + local -A current past window_has_data + local i rows stream current_size past_size growth growth_gb sort_key show + local -a columns + + for i in 0 1 2; do + if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then + echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2 + exit 1 + fi + [[ -z "$rows" ]] && continue + window_has_data[$i]=1 + while IFS='|' read -r stream current_size past_size; do + current[$stream]=$current_size + past[$i,$stream]=$past_size + done <<< "$rows" + done + + printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" + printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + + for stream in "${!current[@]}"; do + columns=() + sort_key=0 + show=false + for i in 0 1 2; do + if [[ -z "${window_has_data[$i]}" ]]; then + columns+=("n/a") + continue + fi + current_size=${current[$stream]} + past_size=${past[$i,$stream]-$current_size} + growth=$(( current_size > past_size ? current_size - past_size : 0 )) + growth_gb=$(bytes_to_gb "$growth") + [[ "$growth_gb" != "0.00" ]] && show=true + [[ $i -eq 0 ]] && sort_key=$growth + columns+=("$growth_gb") + done + if [[ "$show" == true ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}" + fi + done | sort -t'|' -k1,1nr | cut -d'|' -f2- +} + +if ! TG_OUTPUT=$(so_telegraf_output); then + echo "Unable to read telegraf.output from pillar." >&2 + exit 1 +fi + +if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then + so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1 + postgres_indices_growth +else + influxdb_indices_growth +fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate b/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate index 1eb1e33ca..3c501434e 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate @@ -58,6 +58,9 @@ Usage: $(basename "$0") [OPTIONS] Estimate remaining days until Elasticsearch cluster reaches low watermark threshold. +Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to +postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB. + OPTIONS: --json Output results in JSON format -v, --verbose Show additional output @@ -168,11 +171,25 @@ run_indices_growth() { fi } +fetch_postgres_total_bytes() { + local total + + total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1 + echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }' +} + fetch_total_bytes() { local start="$1" local stop="$2" + local pg_start="$3" + local pg_stop="$4" local range_line + if [ "$TG_OUTPUT" = "POSTGRES" ]; then + fetch_postgres_total_bytes "$pg_start" "$pg_stop" + return + fi + if [ -n "$stop" ]; then range_line=" |> range(start: ${start}, stop: ${stop})" else @@ -208,8 +225,13 @@ fail() { echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured." -ORG_ID=$(lookup_org_id) -[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id" +TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar." +if [ "$TG_OUTPUT" = "POSTGRES" ]; then + pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error" +else + ORG_ID=$(lookup_org_id) + [ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id" +fi cluster_storage_size=0 indexed_storage_source="elasticsearch" @@ -221,10 +243,11 @@ if [ -n "$cluster_storage_size_output" ]; then fi fi -# historical data from influxdb for growth calculation -one_day_total=$(fetch_total_bytes "-25h" "-23h") -seven_day_total=$(fetch_total_bytes "-7d8h" "-7d") -thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d") +# historical index sizes for growth calculation +query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf." +one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error" +seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error" +thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error" # available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate) history_days=0 @@ -244,7 +267,12 @@ elif [ "$one_day_total" -gt 0 ]; then historical_total=$one_day_total fi -[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +if [ "$TG_OUTPUT" = "POSTGRES" ]; then + history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +else + history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +fi +[ "$history_days" -gt 0 ] || fail "$history_unavailable" # Daily growth rate growth_bytes=$(( cluster_storage_size - historical_total )) @@ -729,6 +757,7 @@ if [ "$should_trigger_recommendations" = true ]; then [ -n "$index" ] || continue growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}') + [[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0 creation_date_display="" retention_days="" diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0b2bb9bb6..94bf0af81 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -9,6 +9,7 @@ {%- set MDENGINE = GLOBALS.md_engine %} {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %} {%- set TG_OUT = TELEGRAFMERGED.output | upper %} +{%- set PG_ONLY = TG_OUT == 'POSTGRES' %} {%- set PG_HOST = GLOBALS.manager_ip %} {#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar, written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up @@ -149,7 +150,12 @@ # Read metrics about cpu usage [[inputs.cpu]] ## Whether to report per-cpu stats or not +{%- if PG_ONLY %} + {#- SOC and so-stats-show read only cpu-total #} + percpu = false +{%- else %} percpu = true +{%- endif %} ## Whether to report total system cpu stats or not totalcpu = true ## If true, collect raw CPU time metrics. @@ -199,6 +205,11 @@ ## The typical use case is for LVM volumes, to get the VG/LV name instead of ## the near-meaningless DM-0 name. # name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"] +{%- if PG_ONLY %} + {#- Whole disks, md RAID included, already carry the I/O of their partitions and dm devices #} + [inputs.diskio.tagdrop] + name = ["sd*[0-9]", "vd*[0-9]", "xvd*[0-9]", "hd*[0-9]", "nvme*p[0-9]*", "mmcblk*p[0-9]*", "dm-*", "sr*", "loop*"] +{%- endif %} # Get kernel statistics from /proc/stat @@ -242,6 +253,12 @@ password = "{{ ES_PASS }}" insecure_skip_verify = true cluster_health = true +{%- if PG_ONLY %} + {#- SOC reads cluster status, indices and clusterstats_nodes ingest stats; alarms plan jvm and fs #} + cluster_health_level = "cluster" + node_stats = ["indices", "jvm", "fs"] + namedrop = ["elasticsearch_clusterstats_indices"] +{%- endif %} {%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] @@ -249,6 +266,10 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true +{%- if PG_ONLY %} + {#- SOC reads indices; alarms plan jvm and fs #} + node_stats = ["indices", "jvm", "fs"] +{%- endif %} {%- endif %} # ## Timeout for HTTP requests to the elastic search server(s) @@ -291,6 +312,10 @@ collect = ["pipelines"] username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" +{%- if PG_ONLY %} + {#- SOC reads logstash_events only #} + namedrop = ["logstash_plugins"] +{%- endif %} {%- endif %} {% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} @@ -392,6 +417,12 @@ # # Read metrics about network interface usage [[inputs.net]] +{%- if PG_ONLY %} +{%- from 'docker/docker.map.jinja' import DOCKERMERGED %} + {#- Only virtual interfaces; every physical NIC, bond members included, is kept for SOC and alarms #} + [inputs.net.tagdrop] + interface = {{ (['veth*', 'docker*', 'lo'] + DOCKERMERGED.networks.keys() | list) | json }} +{%- endif %} # Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently {%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %} From 65816ae7171698d9e639d6f00d9cd6877848b096 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 18 Nov 2026 10:27:34 -0500 Subject: [PATCH 16/17] FIX: hold push dispatch while salt-master has only just started A push dispatched seconds after a salt-master restart was lost: minions rejoin the new publisher on a randomized ZeroMQ reconnect delay (up to ~22s with the default recon_default/recon_max), and a job published before then reaches nobody. The orchestration's refresh_pillar step timed out with "No minions responded", the drainer logged "push failed" as an ERROR, and so-log-check failed the run. Seen on a standalone 3.5.0 install: a highstate restarted salt-master (new reactor/engine config), the drainer dispatched a soc push 8s later, and the job cache shows the refresh_pillar job matched the minion but got no return. Measured on the same host over five restarts, jobs published in the first 4-9s were lost every time. The drainer now leaves ready intents in place for a later pass while /var/run/salt-master.pid is less than 45s old. On a multi-node grid this also avoids a batch state run silently skipping the minions that have not reconnected yet, since batch only sends the job to minions that answer its ping. --- salt/manager/tools/sbin/so-push-drainer | 19 ++++++++++++++++ .../tools/sbin/so-push-drainer_test.py | 22 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 569f47ef2..aad62b5c7 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -17,6 +17,7 @@ is older than debounce_seconds, this script: * concatenates the actions lists from every ready intent * dedupes by (state or __highstate__, tgt, tgt_type) * drops actions whose target matches no accepted minion (e.g. a role with no nodes yet) + * holds everything for a later pass while salt-master has only just started * dispatches a single `salt-run state.orchestrate orch.push_batch --async` with the deduped actions list passed as pillar kwargs * deletes the contributed intent files on successful dispatch @@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending' LOCK_FILE = os.path.join(PENDING_DIR, '.lock') LOG_FILE = '/opt/so/log/salt/so-push-drainer.log' MASTER_CONFIG = '/etc/salt/master' +MASTER_PIDFILE = '/var/run/salt-master.pid' + +# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval +# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody. +# Doubled to allow for a slow master start. +MASTER_SETTLE_SECONDS = 45 DISPATCHED_DIR = '/opt/so/state/push_dispatched' @@ -140,6 +147,14 @@ def _targeted(actions, log): return targeted +def _master_settling(now): + try: + age = now - os.path.getmtime(MASTER_PIDFILE) + except OSError: + return False + return 0 <= age < MASTER_SETTLE_SECONDS + + def _dispatch(actions, log): pillar_arg = json.dumps({'actions': actions}) cmd = [ @@ -410,6 +425,10 @@ def main(): log.debug('no ready intents (%d still in debounce window)', skipped) return 0 + if _master_settling(now): + log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready)) + return 0 + combined_actions = [] oldest_first_touch = now all_paths = [] diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index a6deab5f2..3f26d1822 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase): drainer._unlink(self.tmpdir, self.log) self.log.exception.assert_called_once() + def test_master_settling(self): + pidfile = os.path.join(self.tmpdir, 'salt-master.pid') + now = time.time() + with patch.object(drainer, 'MASTER_PIDFILE', pidfile): + self.assertFalse(drainer._master_settling(now)) + self.write_json(self.tmpdir, 'salt-master.pid', '1100') + for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True), + (drainer.MASTER_SETTLE_SECONDS, False), (-60, False)): + os.utime(pidfile, (now - age, now - age)) + self.assertIs(drainer._master_settling(now), settling, age) + class TestDispatch(DrainerTestCase): @@ -475,6 +486,7 @@ class TestMain(DrainerTestCase): ('_load_push_cfg', {'side_effect': lambda: self.cfg}), ('_check_dispatched', {}), ('_targeted', {'side_effect': lambda actions, log: actions}), + ('_master_settling', {'return_value': False}), ): patcher = patch.object(drainer, name, **kwargs) setattr(self, name, patcher.start()) @@ -528,6 +540,16 @@ class TestMain(DrainerTestCase): with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')): self.assertEqual(drainer.main(), 0) + def test_holds_while_master_settling(self): + path = self.intent('pillar_soc.json') + self._master_settling.return_value = True + with patch.object(drainer, '_dispatch') as dispatch: + self.assertEqual(drainer.main(), 0) + dispatch.assert_not_called() + self._check_dispatched.assert_called_once() + self.assertTrue(os.path.exists(path)) + self.assertIn('holding 1 intent(s)', self.logged('info')) + def test_dispatch_failure_keeps_intents(self): path = self.intent('pillar_soc.json') with patch.object(drainer, '_dispatch', return_value=None): From aba2f5ea430228d77b778dc45b53b9f2b125019e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 18 Nov 2026 10:27:34 -0500 Subject: [PATCH 17/17] FIX: retry the push orchestration's refresh_pillar step Holding dispatch in the drainer cannot help an orchestration that is already running when salt-master restarts: its first action raises, and the next action's refresh_pillar is published seconds after the restart, before minions have rejoined the publisher, so it fails with "No minions responded". refresh_pillar now retries (3 attempts, 10s apart; a failed attempt takes about 15s to time out, so the later attempts land after the reconnect window). It also runs ahead of highstate actions, which now require it, so they get the same reachability check. Retrying the highstate step itself is not an option because it would re-run a fleetwide highstate whenever any state failed. Run straight after a salt-master restart on a standalone 3.5.0 install, the orchestration previously failed with "No minions responded"; it now succeeds on the second attempt. A target that is genuinely offline takes about 65s to fail instead of 15s. --- salt/orch/push_batch.sls | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/salt/orch/push_batch.sls b/salt/orch/push_batch.sls index 2fff2f646..97ffba048 100644 --- a/salt/orch/push_batch.sls +++ b/salt/orch/push_batch.sls @@ -4,8 +4,19 @@ {% set BATCH_WAIT = AUTOAPPLY.batch_wait %} {# queue must be a top-level salt.state arg (kwarg is ignored); an int is max_queue and still fails on conflict #} +{# refresh_pillar doubles as the reachability check: minions take up to ~22s to rejoin a restarted master's + publisher, and a job published before then reaches nobody. Each failed attempt takes ~15s to time out. #} {% for action in actions %} +refresh_pillar_{{ loop.index }}: + salt.function: + - name: saltutil.refresh_pillar + - tgt: '{{ action.tgt }}' + - tgt_type: {{ action.get('tgt_type', 'compound') }} + - retry: + attempts: 3 + interval: 10 + {% if action.get('highstate') %} apply_highstate_{{ loop.index }}: salt.state: @@ -15,13 +26,9 @@ apply_highstate_{{ loop.index }}: - batch: {{ action.get('batch', BATCH) }} - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }} - queue: True + - require: + - salt: refresh_pillar_{{ loop.index }} {% else %} -refresh_pillar_{{ loop.index }}: - salt.function: - - name: saltutil.refresh_pillar - - tgt: '{{ action.tgt }}' - - tgt_type: {{ action.get('tgt_type', 'compound') }} - apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}: salt.state: - tgt: '{{ action.tgt }}'