Postgres: run pg_partman maintenance over the local socket and detect failed runs

salt/common/tools/sbin/so-status

@@ -17,6 +17,7 @@ import time
 EXPECTED_CONTAINERS_FILE = "/opt/so/conf/so-status/so-status.conf"
 HIGHSTATE_COMPLETE_FILE = "/opt/so/log/salt/lasthighstate"
 UPTIME_FILE = "/proc/uptime"
+POSTGRES_MAINTENANCE_STATUS_FILE = "/opt/so/log/postgres/maintenance.status"
 
 def showUsage(options, args):
   print('Usage: {} [OPTIONS]'.format(sys.argv[0]))
@@ -149,6 +150,20 @@ def check_container_status(options, console):
   return code, container_list
 
 
+def check_postgres_maintenance(code, container_list):
+  try:
+    with open(POSTGRES_MAINTENANCE_STATUS_FILE, "r") as status_file:
+      status = status_file.readline().strip()
+  except OSError:
+    return code
+  if status.startswith("failed"):
+    reason = status[len("failed"):].lstrip(":").strip()
+    container_list.append({ "Name": "so-postgres-maintenance", "Status": "failed", "Details": reason })
+    if code == 0:
+      code = 1
+  return code
+
+
 def check_status(options, console):
   container_list = []
   code = check_installation_status(options, console)
@@ -160,6 +175,7 @@ def check_status(options, console):
   # on the Grid until a highstate completes.
   if code == 0 or code == 2:
     container_code, container_list = check_container_status(options, console)
+    container_code = check_postgres_maintenance(container_code, container_list)
     if code == 0:
       code = container_code
   output(options, console, code, container_list)

salt/docker/defaults.yaml

@@ -103,6 +103,7 @@ docker:
         - 80:80
         - 443:443
         - 8443:8443
+        - 7788:7788
         - 7789:7789
       custom_bind_mounts: []
       extra_hosts: []

salt/firewall/defaults.yaml

@@ -118,6 +118,10 @@ firewall:
       tcp:
         - 5601
       udp: []
+    localrules:
+      tcp:
+        - 7788
+      udp: []
     nginx:
       tcp:
         - 80
@@ -196,6 +200,7 @@ firewall:
                 - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
+                - localrules
             sensor:
               portgroups:
                 - beats_5044
@@ -386,6 +391,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
                 - sensoroni
             fleet:
               portgroups:
@@ -605,6 +611,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
                 - sensoroni
             fleet:
               portgroups:
@@ -822,6 +829,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
                 - sensoroni
             fleet:
               portgroups:
@@ -1050,6 +1058,7 @@ firewall:
                 - elastic_agent_update
                 - endgame
                 - strelka_frontend
+                - localrules
             fleet:
               portgroups:
                 - elasticsearch_rest
@@ -1509,6 +1518,7 @@ firewall:
                 - elasticsearch_rest
                 - elasticsearch_node
                 - elastic_agent_control
+                - localrules
             sensor:
               portgroups:
                 - beats_5044

salt/firewall/soc_firewall.yaml

@@ -138,6 +138,9 @@ firewall:
     kibana:
       tcp: *tcpsettings
       udp: *udpsettings
+    localrules:
+      tcp: *tcpsettings
+      udp: *udpsettings
     nginx:
       tcp: *tcpsettings
       udp: *udpsettings

salt/manager/tools/sbin/so-yaml.py

@@ -20,7 +20,6 @@ def showUsage(args):
     print('    append           - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr)
     print('    removelistitem   - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
-    print('    purgelistitem    - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    replacelistobject  - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
     print('    add              - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
     print('    get [-r]         - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
@@ -200,33 +199,6 @@ def removelistitem(args):
     return 0
 
 
-def purgeListItem(content, listItem):
-    if isinstance(content, dict):
-        for value in content.values():
-            purgeListItem(value, listItem)
-    elif isinstance(content, list):
-        content[:] = [item for item in content if item != listItem]
-        for item in content:
-            purgeListItem(item, listItem)
-
-
-def purgelistitem(args):
-    if len(args) != 3:
-        print('Missing filename, key arg, or list item to purge', file=sys.stderr)
-        showUsage(None)
-        return 1
-
-    filename = args[0]
-    key = args[1]
-    listItem = args[2]
-
-    content = loadYaml(filename)
-    purgeListItem(getKeyValue(content, key), convertType(listItem))
-    writeYaml(filename, content)
-
-    return 0
-
-
 def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
@@ -413,7 +385,6 @@ def main():
         "append": append,
         "appendlistobject": appendlistobject,
         "removelistitem": removelistitem,
-        "purgelistitem": purgelistitem,
         "replacelistobject": replacelistobject,
         "get": get,
         "remove": remove,

salt/manager/tools/sbin/so-yaml_test.py

@@ -709,223 +709,6 @@ class TestRemoveListItem(unittest.TestCase):
                 self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
 
-class TestPurgeListItem(unittest.TestCase):
-
-    def test_purgelistitem_missing_arg(self):
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd", "help"]
-                soyaml.purgelistitem(["file", "key"])
-                sysmock.assert_called()
-                self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue())
-
-    def test_purgelistitem(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [a,b,c], key2: false}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- c\nkey2: false\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_nested_lists(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - c\n  child2:\n    deep1:\n    - d\n    deep2: 45\nkey2:\n- b\n- e\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_duplicates(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [b,a,b,c,b]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_list_of_objects(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- name: item1\n  values:\n  - a\n- name: item2\n  values:\n  - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_typed(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [1, '1', 2]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "1"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- '1'\n- 2\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_item_not_in_list(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c] }}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "d"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - b\n  - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_key_noexist(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [a,b,c]}")
-        file.close()
-
-        result = soyaml.purgelistitem([filename, "key2.child1", "b"])
-        self.assertEqual(result, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- b\n- c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_key_noexist_deep(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c] }}")
-        file.close()
-
-        result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"])
-        self.assertEqual(result, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - b\n  - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_scalar_and_null_values(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: b, child2: null, child3: [b] }}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1: b\n  child2: null\n  child3: []\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_lists_of_lists(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [[a,b], [b,[b,c]], b]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- - a\n- - - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_file_object(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}")
-        file.close()
-
-        itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml"
-        file = open(itemfile, "w")
-        file.write("name: x\n")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "file:" + itemfile])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- name: y\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_main(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c] }, key2: [b]}")
-        file.close()
-
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"]
-            soyaml.main()
-            sysmock.assert_called_once_with(0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - c\nkey2:\n- b\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_with_remove(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}")
-        file.close()
-
-        soyaml.remove([filename, "key1.child1"])
-        soyaml.purgelistitem([filename, "key2", "child1"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child2:\n  - b\nkey2:\n  group1:\n    items:\n    - x\n    - y\n  group2:\n    items: []\n"
-        self.assertEqual(actual, expected)
-
-
 class TestAppendListObject(unittest.TestCase):
 
     def test_appendlistobject_missing_arg(self):

salt/manager/tools/sbin/soup

@@ -527,7 +527,6 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
     [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
     [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
-    [[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0
     true
 }
 
@@ -547,6 +546,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
     [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
+    [[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0
     # All applicable post-upgrade steps completed; clear the resume marker.
     rm -f "$POSTVERSION_FILE"
     true
@@ -1070,11 +1070,14 @@ telegraf_repair() {
 
   echo "Checking Telegraf metric partitions."
   local status=0
-  "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
+  local max_age
+  max_age=$(lookup_pillar maintenance_max_age_hours postgres:telegraf 2>/dev/null) || true
+  max_age=${max_age:-3}
+  MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
   case "$status" in
     0) echo "  Telegraf partitions are healthy; nothing to repair." ;;
     1) echo "  Repairing stalled Telegraf partitions."
-       "$repair" --yes \
+       MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --yes \
          || echo "  warning: so-telegraf-repair failed; run it manually" >&2 ;;
     *) echo "  Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
   esac
@@ -1204,21 +1207,10 @@ add_detection_runtime_fields() {
 ### 3.4.0 End ###
 
 ### 3.5.0 Scripts ###
-up_to_3.5.0() {
-  remove_localrules_portgroup
+post_to_3.5.0() {
+  telegraf_repair
 
-  INSTALLEDVERSION=3.5.0
-}
-
-remove_localrules_portgroup() {
-  local pillar_file
-
-  for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do
-    grep -qs localrules "$pillar_file" || continue
-    echo "Removing the localrules portgroup from $pillar_file."
-    so-yaml.py remove "$pillar_file" firewall.portgroups.localrules
-    so-yaml.py purgelistitem "$pillar_file" firewall.role localrules
-  done
+  set_postversion 3.5.0
 }
 ### 3.5.0 End ###
 

salt/nginx/enabled.sls

@@ -54,9 +54,9 @@ so-nginx:
       - /opt/so/log/nginx/:/var/log/nginx:rw
       - /opt/so/tmp/nginx/:/var/lib/nginx:rw
       - /opt/so/tmp/nginx/:/run:rw
-      - /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro
-      {% if GLOBALS.is_manager %}
       - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
+      - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts 
+      {% if GLOBALS.is_manager %}
       - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
       - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
       # ATT&CK Navigator binds
@@ -64,6 +64,7 @@ so-nginx:
       - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
       - /nsm/repo:/opt/socore/html/repo:ro
       - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
+      - /nsm/rules:/nsm/rules:ro
       {%   if NGINXMERGED.external_suricata %}
       - /opt/so/rules/nids/suri:/surirules:ro
       {%   endif %}

salt/nginx/etc/nginx.conf

@@ -4,8 +4,6 @@
 {%- set role = grains.id.split('_') | last %}
 {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
 {%- set influxauth = ('so:' + influxpass) | base64_encode %}
-{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
-{%- set full_server_name = all_names | unique | join(' ') %}
 
 worker_processes auto;
 error_log /var/log/nginx/error.log;
@@ -41,31 +39,52 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 
+	{%- if role in ['fleet'] %}
+
+	server {
+		listen 8443;
+		server_name {{ GLOBALS.hostname }};
+		root /opt/socore/html;
+		location /artifacts/ {
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
+		}
+	}
+
+    {%- endif %}
+
 	{%- if GLOBALS.is_manager %}
+	{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
+	{%- set full_server_name = all_names | unique | join(' ') %}
+
 	server {
 		listen 80 default_server;
 		server_name _;
 		return 307 https://{{ GLOBALS.url_base }}$request_uri;
 	}
-  {%- endif %}
-  {%- if GLOBALS.is_manager or role == 'fleet' %}
+
 	server {
 		listen 8443;
-    {%- if GLOBALS.is_manager %}
 		server_name {{ full_server_name }};
-    {%- else %}
-    server_name {{ GLOBALS.hostname }};
-    {%- endif %}
-		root /opt/socore/elastic-artifacts;
+		root /opt/socore/html;
 		location /artifacts/ {
-			try_files $uri =404;
-		}
-		location / {
-			return 404;
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
 		}
 	}
-  {%- endif %}
-  {%- if GLOBALS.is_manager %}
+
 	server {
 		listen 443 ssl default_server;
 		http2 on;
@@ -92,6 +111,21 @@ http {
 	{%- endif %}
 
 	{%- if GLOBALS.is_manager %}
+
+	server {
+		listen 7788;
+		server_name {{ full_server_name }};
+		root /nsm/rules;
+		location / {
+			allow all;
+			sendfile on;
+			sendfile_max_chunk 1m;
+			autoindex on;
+			autoindex_exact_size off;
+			autoindex_format html;
+			autoindex_localtime on;
+		}
+	}
 	{%-   if NGINXMERGED.external_suricata %}
     server {
         listen 7789 ssl;

salt/postgres/defaults.yaml

@@ -2,6 +2,7 @@ postgres:
   enabled: True
   telegraf:
     retention_days: 14
+    maintenance_max_age_hours: 3
   config:
     listen_addresses: '*'
     port: 5432
@@ -17,3 +18,6 @@ postgres:
     log_min_messages: 'warning'
     shared_preload_libraries: pg_cron
     cron.database_name: postgres
+    # pg_hba.conf only trusts local socket connections; over TCP pg_cron has no
+    # password to send and every job fails to connect.
+    cron.host: /run/postgresql

salt/postgres/disabled.sls

@@ -24,6 +24,15 @@ so_postgres_backup:
     - identifier: so_postgres_backup
     - user: root
 
+so_postgres_maintenance_status:
+  cron.absent:
+    - identifier: so_postgres_maintenance_status
+    - user: root
+
+so_postgres_maintenance_status_file:
+  file.absent:
+    - name: /opt/so/log/postgres/maintenance.status
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 {%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
 
 include:
@@ -122,6 +123,29 @@ so_postgres_backup:
     - month: '*'
     - dayweek: '*'
 
+{% if TELEGRAFMERGED.output | upper in ['POSTGRES', 'BOTH'] %}
+{%   set max_age = salt['pillar.get']('postgres:telegraf:maintenance_max_age_hours', 3) | int %}
+so_postgres_maintenance_status:
+  cron.present:
+    - name: MAINTENANCE_MAX_AGE_HOURS={{ max_age }} /usr/sbin/so-telegraf-postgres status > /dev/null 2>&1
+    - identifier: so_postgres_maintenance_status
+    - user: root
+    - minute: '*/15'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{% else %}
+so_postgres_maintenance_status:
+  cron.absent:
+    - identifier: so_postgres_maintenance_status
+    - user: root
+
+so_postgres_maintenance_status_file:
+  file.absent:
+    - name: /opt/so/log/postgres/maintenance.status
+{% endif %}
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/soc_postgres.yaml

@@ -9,6 +9,11 @@ postgres:
       description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
       forcedType: int
       helpLink: postgres
+    maintenance_max_age_hours:
+      description: Number of hours without a successful run of the hourly pg_partman maintenance job before the Telegraf metrics database is reported as unhealthy in grid status and repaired by soup.
+      forcedType: int
+      advanced: True
+      helpLink: postgres
   config:
     max_connections:
       description: Maximum number of concurrent PostgreSQL connections.
@@ -87,3 +92,8 @@ postgres:
       global: True
       advanced: True
       helpLink: postgres
+    cron.host:
+      description: Unix socket directory pg_cron uses to run scheduled jobs. Must be a local socket directory so jobs authenticate with the local trust rule; a TCP host requires a password and jobs will fail.
+      global: True
+      advanced: True
+      helpLink: postgres

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -12,7 +12,11 @@ set -e
 #   retention    Reconcile partman retention and premake on telegraf parents.
 #                Env: RETENTION_DAYS.
 #   maintenance  Drain default partitions and run partman maintenance.
-#   check        Report partition health. Non-zero if any parent is unhealthy.
+#   check        Report partition and maintenance health. Non-zero if any parent
+#                is unhealthy or the maintenance job has no successful run within
+#                the last MAINTENANCE_MAX_AGE_HOURS (default 3).
+#   status       Run check and write `ok` or `failed: <reason>` to
+#                /opt/so/log/postgres/maintenance.status for so-status.
 #
 # A default partition holding rows for a day blocks creating that day's child,
 # so maintenance drains defaults before calling partman. Use so-telegraf-repair
@@ -189,6 +193,11 @@ SELECT cron.schedule_in_database(
   'CALL so_admin.telegraf_maintenance()',
   'so_telegraf'
 );
+-- pg_cron copies cron.host into nodename only when a job is first created, so
+-- jobs registered before cron.host changed keep connecting to the old host.
+UPDATE cron.job SET nodename = current_setting('cron.host')
+ WHERE jobname = 'telegraf-partman-maintenance'
+   AND nodename IS DISTINCT FROM current_setting('cron.host');
 \else
 \echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
 \endif
@@ -278,16 +287,80 @@ LIMIT 5;
 \echo 'pg_cron is not installed in this database.'
 \endif
 EOSQL
+    max_age="${MAINTENANCE_MAX_AGE_HOURS:-3}"
+    [[ "$max_age" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 1; }
+    failures=()
     unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
       "SELECT count(*) FROM so_admin.telegraf_partition_status()
        WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
     if [ "${unhealthy:-1}" != "0" ]; then
-      echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
+      failures+=("$unhealthy telegraf parent(s) unhealthy")
+    fi
+    has_cron=$(docker exec so-postgres psql -U postgres -d postgres -tAc \
+      "SELECT to_regclass('cron.job_run_details') IS NOT NULL")
+    if [ "$has_cron" != "t" ]; then
+      failures+=("pg_cron is not installed in the postgres database")
+    else
+      runs=$(docker exec so-postgres psql -U postgres -d postgres -tA -F '|' -c \
+        "SELECT (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance'),
+                (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance' AND active),
+                (SELECT count(*) FROM pg_stat_activity
+                  WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'),
+                (SELECT count(*) FROM pg_stat_activity
+                  WHERE (backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%')
+                    AND backend_start < now() - make_interval(hours => $max_age)),
+                count(*),
+                count(*) FILTER (WHERE d.status = 'succeeded'
+                                   AND d.start_time > now() - make_interval(hours => $max_age)),
+                coalesce((array_agg(format('last run %s at %s: %s', d.status, d.start_time,
+                                           regexp_replace(coalesce(d.return_message, ''), '\s+', ' ', 'g'))
+                                    ORDER BY d.start_time DESC))[1], '')
+           FROM cron.job j
+           JOIN cron.job_run_details d ON d.jobid = j.jobid
+          WHERE j.jobname = 'telegraf-partman-maintenance'") || runs=""
+      IFS='|' read -r jobs active launchers old_launchers run_count recent_ok last_run <<< "$runs"
+      # With no runs recorded, a live launcher younger than N hours means the
+      # job was only just scheduled (fresh install or a restart for a config
+      # change) and has not reached its first :17 yet. A launcher older than
+      # that should have fired it by now. Tradeoff: a job first scheduled long
+      # after the launcher started reports failed until its first run.
+      if [ -z "$runs" ]; then
+        failures+=("could not read telegraf-partman-maintenance runs from cron.job_run_details")
+      elif [ "$jobs" = "0" ]; then
+        failures+=("telegraf-partman-maintenance is not scheduled")
+      elif [ "$active" = "0" ]; then
+        failures+=("telegraf-partman-maintenance is inactive")
+      elif [ "$launchers" = "0" ]; then
+        failures+=("pg_cron launcher is not running; telegraf-partman-maintenance cannot run")
+      elif [ "$run_count" = "0" ]; then
+        if [ "$old_launchers" != "0" ]; then
+          failures+=("telegraf-partman-maintenance has never run although pg_cron has been up for more than $max_age hour(s)")
+        fi
+      elif [ "$recent_ok" = "0" ]; then
+        failures+=("no successful telegraf-partman-maintenance run in the last $max_age hour(s); $last_run")
+      fi
+    fi
+    if [ "${#failures[@]}" -gt 0 ]; then
+      msg=$(printf '%s; ' "${failures[@]}")
+      echo "so-telegraf-postgres check: ${msg%; }" >&2
       exit 1
     fi
     echo "so-telegraf-postgres check: all telegraf parents healthy"
     ;;
 
+  status)
+    status_file=/opt/so/log/postgres/maintenance.status
+    tmp=$(mktemp "$status_file.XXXXXX")
+    if reason=$("$0" check 2>&1 >/dev/null); then
+      echo ok > "$tmp"
+    else
+      reason=$(printf '%s\n' "$reason" | tail -n 1)
+      echo "failed: ${reason#so-telegraf-postgres check: }" > "$tmp"
+    fi
+    chmod 644 "$tmp"
+    mv -f "$tmp" "$status_file"
+    ;;
+
   *)
     echo "Unknown subcommand: $cmd" >&2
     exit 1

salt/postgres/tools/sbin/so-telegraf-repair

@@ -15,6 +15,9 @@
 #   --yes         Skip the confirmation prompt (for soup and other automation).
 #   --no-restart  Never restart so-postgres, even if pg_cron's launcher is dead.
 #
+# Env: MAINTENANCE_MAX_AGE_HOURS (default 3) is how long the maintenance job may
+# go without a successful run before repair is needed.
+#
 # Exit status:
 #   0  healthy, or repair completed
 #   1  repair is needed (--check only)
@@ -25,6 +28,8 @@ set -e
 # Matches p_premake in telegraf.conf's create_parent template.
 PREMAKE=7
 JOB_NAME=telegraf-partman-maintenance
+MAX_AGE_HOURS="${MAINTENANCE_MAX_AGE_HOURS:-3}"
+[[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 2; }
 
 CHECK_ONLY=false
 ASSUME_YES=false
@@ -118,14 +123,30 @@ launcher=$(psql_pg -tAc \
 # so_telegraf before the postgres state lands, postgres after.
 cron_db=$(docker exec so-postgres psql -U postgres -tAc \
   "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
+run_count=0
+recent_ok=0
 last_run=never
-if [[ -n "$cron_db" ]]; then
-  last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
-    "SELECT coalesce(max(d.start_time)::text, 'never')
+has_run_details=$([[ -n "$cron_db" ]] && docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
+  "SELECT to_regclass('cron.job_run_details') IS NOT NULL" 2>/dev/null || echo f)
+if [[ "$has_run_details" == t ]]; then
+  runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tA -F '|' -c \
+    "SELECT count(*),
+            count(*) FILTER (WHERE d.status = 'succeeded'
+                               AND d.start_time > now() - make_interval(hours => $MAX_AGE_HOURS)),
+            coalesce((array_agg(format('%s at %s', d.status, d.start_time)
+                                ORDER BY d.start_time DESC))[1], 'never')
        FROM cron.job_run_details d JOIN cron.job j USING (jobid)
-      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
-  [[ -n "$last_run" ]] || last_run=never
+      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null) \
+    || { echo "Could not read $JOB_NAME runs from cron.job_run_details; cannot judge maintenance health." >&2; exit 2; }
+  IFS='|' read -r run_count recent_ok last_run <<< "$runs"
 fi
+# An unreadable run history exits 2 rather than 1: soup repairs on 1, which
+# truncates the default partitions, and a transient query error must not do that.
+# No runs at all means a freshly scheduled job (or pg_cron not loaded, handled
+# by restart_needed); the partition checks catch a job that never fires. Runs
+# with no recent success mean the job fires but fails, e.g. cannot connect.
+maintenance_failing=false
+[[ "$run_count" -gt 0 && "$recent_ok" -eq 0 ]] && maintenance_failing=true
 
 # A grid that has never written a metric has nothing to recover, and an empty
 # cron_db means pg_cron is not loaded at all, which no restart fixes.
@@ -137,6 +158,7 @@ repair_needed=false
 [[ "$behind" -gt 0 ]] && repair_needed=true
 [[ "$misconfigured" -gt 0 ]] && repair_needed=true
 $restart_needed && repair_needed=true
+$maintenance_failing && repair_needed=true
 
 echo "Telegraf partition status:"
 psql_tg -c "$REPORT"
@@ -157,6 +179,7 @@ if $CHECK_ONLY; then
   [[ "$behind" -gt 0 ]]      && echo "  * $behind parent(s) with no partition for the current window"
   [[ "$misconfigured" -gt 0 ]] && echo "  * $misconfigured parent(s) with stale partman settings"
   $restart_needed            && echo "  * pg_cron's launcher is dead; maintenance is not running at all"
+  $maintenance_failing       && echo "  * no successful $JOB_NAME run in the last $MAX_AGE_HOURS hour(s)"
   echo
   echo "Re-run without --check to repair."
   exit 1
@@ -241,5 +264,6 @@ EOSQL
 echo
 echo "Telegraf partition status after repair:"
 psql_tg -c "$REPORT"
-echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
+echo "The $JOB_NAME job runs hourly at :17. Until its next run succeeds, grid status"
+echo "may still report so-postgres-maintenance as failed. Confirm it fired with:"
 echo "  so-telegraf-repair --check"