Merge pull request #15763 from Security-Onion-Solutions/reyesj2-es932

start loading addon integration index templates

VERSION

@@ -1 +1 @@
-3.1.0
+3.0.0-foxtrot

pillar/elasticsearch/index_templates.sls

@@ -1,2 +0,0 @@
-elasticsearch:
-  index_settings:

pillar/top.sls

@@ -38,9 +38,6 @@ base:
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -63,8 +60,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -102,13 +97,9 @@ base:
     - node_data.ips
     - secrets
     - healthcheck.eval
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -134,8 +125,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - backup.soc_backup
     - backup.adv_backup
     - zeek.soc_zeek
@@ -152,13 +141,9 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -173,8 +158,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -271,13 +254,9 @@ base:
   '*_import':
     - node_data.ips
     - secrets
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -303,8 +282,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - zeek.soc_zeek
     - zeek.adv_zeek
     - bpf.soc_bpf

salt/allowed_states.map.jinja

@@ -29,8 +29,6 @@
     'manager',
     'nginx',
     'influxdb',
-    'postgres',
-    'postgres.auth',
     'soc',
     'kratos',
     'hydra',

salt/backup/config_backup.sls

@@ -32,23 +32,3 @@ so_config_backup:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
-
-postgres_backup_script:
-  file.managed:
-    - name: /usr/sbin/so-postgres-backup
-    - user: root
-    - group: root
-    - mode: 755
-    - source: salt://backup/tools/sbin/so-postgres-backup
-
-# Add postgres database backup
-so_postgres_backup:
-  cron.present:
-    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
-    - identifier: so_postgres_backup
-    - user: root
-    - minute: '5'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'

salt/backup/tools/sbin/so-postgres-backup

@@ -1,36 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-TODAY=$(date '+%Y_%m_%d')
-BACKUPDIR=/nsm/backup
-BACKUPFILE="$BACKUPDIR/so-postgres-backup-$TODAY.sql.gz"
-MAXBACKUPS=7
-
-mkdir -p $BACKUPDIR
-
-# Skip if already backed up today
-if [ -f "$BACKUPFILE" ]; then
-  exit 0
-fi
-
-# Skip if container isn't running
-if ! docker ps --format '{{.Names}}' | grep -q '^so-postgres$'; then
-  exit 0
-fi
-
-# Dump all databases and roles, compress
-docker exec so-postgres pg_dumpall -U postgres | gzip > "$BACKUPFILE"
-
-# Retention cleanup
-NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
-while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
-  OLDEST=$(find $BACKUPDIR -type f -name "so-postgres-backup*" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
-  rm -f "$OLDEST"
-  NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
-done

salt/ca/files/signing_policies.conf

@@ -54,20 +54,6 @@ x509_signing_policies:
     - extendedKeyUsage: serverAuth
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
-  postgres:
-    - minions: '*'
-    - signing_private_key: /etc/pki/ca.key
-    - signing_cert: /etc/pki/ca.crt
-    - C: US
-    - ST: Utah
-    - L: Salt Lake City
-    - basicConstraints: "critical CA:false"
-    - keyUsage: "critical keyEncipherment"
-    - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid,issuer:always
-    - extendedKeyUsage: serverAuth
-    - days_valid: 820
-    - copypath: /etc/pki/issued_certs/
   elasticfleet:
     - minions: '*'
     - signing_private_key: /etc/pki/ca.key

salt/common/tools/sbin/so-image-common

@@ -31,7 +31,6 @@ container_list() {
       "so-hydra"
       "so-nginx"
       "so-pcaptools"
-      "so-postgres"
       "so-soc"
       "so-suricata"
       "so-telegraf"
@@ -56,7 +55,6 @@ container_list() {
       "so-logstash"
       "so-nginx"
       "so-pcaptools"
-      "so-postgres"
       "so-redis"
       "so-soc"
       "so-strelka-backend"

salt/docker/defaults.yaml

@@ -237,11 +237,3 @@ docker:
       extra_hosts: []
       extra_env: []
       ulimits: []
-    'so-postgres':
-      final_octet: 89
-      port_bindings:
-        - 0.0.0.0:5432:5432
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
-      ulimits: []

salt/elasticfleet/content-defaults.map.jinja

@@ -0,0 +1,123 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
+
+{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core content packages #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each content package #}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
+{%       for pattern in pkg.dataStreams %}
+{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
+             title: generic is an artifact of that and is not in use #}
+{%         if pattern.title == "generic" %}
+{%           continue %}
+{%         endif %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{#         on content integrations the component name is user defined at the time it is added to an agent policy #}
+{%         set component_name = pattern.title %}
+{%         set index_pattern = pattern.name %}
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
+{#           Default integration settings #}
+{%           set integration_defaults = {
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
+                       "set_priority": {"priority": 100}
+                      },
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
+
+{%         do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%       endfor %}
+{%     else %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -9,16 +9,22 @@
   "namespace": "so",
   "description": "Zeek Import logs",
   "policy_id": "so-grid-nodes_general",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/zeek/logs/*.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -34,7 +40,8 @@
             "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -15,19 +15,25 @@
     "version": ""
   },
   "name": "kratos-logs",
+  "namespace": "so",
   "description": "Kratos logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/kratos/kratos.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "kratos",
             "pipeline": "kratos",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -48,10 +54,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -9,16 +9,22 @@
   "namespace": "so",
   "description": "Zeek logs",
   "policy_id": "so-grid-nodes_general",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/zeek/logs/current/*.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "zeek",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
@@ -30,10 +36,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.0.2",
+    "version": "9.3.0",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json

@@ -6,21 +6,23 @@
   "name": "agent-monitor",
   "namespace": "",
   "description": "",
+  "policy_id": "so-grid-nodes_general",
   "policy_ids": [
     "so-grid-nodes_general"
   ],
-  "output_id": null,
   "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/agents/agent-monitor.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "agentmonitor",
             "pipeline": "elasticagent.monitor",
             "parsers": "",
@@ -34,15 +36,16 @@
             "ignore_older": "72h",
             "clean_inactive": -1,
             "harvester_limit": 0,
-            "fingerprint": true,
+            "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": 64,
-            "file_identity_native": false,
+            "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "hydra-logs",
+  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hydra",
             "pipeline": "hydra",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -34,10 +40,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "idh-logs",
+  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "idh",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -4,26 +4,32 @@
     "version": ""
   },
   "name": "import-evtx-logs",
+  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.13.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.6.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.13.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.13.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.6.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "import-suricata-logs",
+  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -4,14 +4,18 @@
     "version": ""
   },
   "name": "rita-logs",
+  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -19,6 +23,8 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "rita",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "so-ip-mappings",
+  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hostnamemappings",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-auth-sync-logs",
+  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -4,20 +4,26 @@
     "version": ""
   },
   "name": "soc-detections-logs",
+  "namespace": "so",
   "description": "Security Onion Console - Detections Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -35,10 +41,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-salt-relay-logs",
+  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-sensoroni-logs",
+  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-server-logs",
+  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "strelka-logs",
+  "namespace": "so",
   "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "strelka",
             "pipeline": "strelka.file",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "suricata-logs",
+  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "suricata",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/input-defaults.map.jinja

@@ -0,0 +1,123 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
+
+{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core input packages #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each input package #}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
+{%       for pattern in pkg.dataStreams %}
+{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
+             title: generic is an artifact of that and is not in use #}
+{%         if pattern.title == "generic" %}
+{%           continue %}
+{%         endif %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{#         on input integrations the component name is user defined at the time it is added to an agent policy #}
+{%         set component_name = pattern.title %}
+{%         set index_pattern = pattern.name %}
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
+{#           Default integration settings #}
+{%           set integration_defaults = {
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
+                       "set_priority": {"priority": 100}
+                      },
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
+
+{%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
+{%       endfor %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/integration-defaults.map.jinja

@@ -59,8 +59,8 @@
 {#     skip core integrations #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each integration #}
-{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
-{%       for pattern in pkg.es_index_patterns %}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
+{%       for pattern in pkg.dataStreams %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
@@ -75,44 +75,27 @@
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
-
-{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
-           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
-{%         set custom_component_name = component_name %}
-
-{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
-{%         set generic_integration_type = integration_type %}
-
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
-{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
-{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
-{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
-{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
-{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
-{%           set component_name = "filestream.generic" %}
-{%           set generic_integration_type = "logs-" %}
-{%         endif %}
-
 {#           Default integration settings #}
 {%           set integration_defaults = {
                "index_sorting": false,
                "index_template": {
-                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                  "data_stream": {
                    "allow_custom_routing": false,
                    "hidden": false
                  },
-                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
                  "index_patterns": [index_pattern],
                  "priority": 501,
                  "template": {
                    "settings": {
                      "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
                          "number_of_replicas": 0
                      }
                    }

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() {
     fi
 }
 
-elastic_fleet_installed_packages() {
-    if ! fleet_api "epm/packages/installed?perPage=500"; then
+elastic_fleet_get_package_list_by_type() {
+    if ! output=$(fleet_api "epm/packages"); then
         return 1
+    else
+        is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
+        is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
+        is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
+        jq -n --argjson is_integration "${is_integration:-[]}" \
+              --argjson is_input "${is_input:-[]}" \
+              --argjson is_content "${is_content:-[]}" \
+              '{"integration": $is_integration,"input": $is_input, "content": $is_content}'
+    fi
+}
+elastic_fleet_installed_packages_components() {
+    package_type=${1,,}
+    if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
+        echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
+        return 1
+    fi
+
+    packages_by_type=$(elastic_fleet_get_package_list_by_type)
+    packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
+
+    if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
+        return 1
+    else
+        jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
     fi
 }
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
-PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
+CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
 COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
@@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
-        # Write out file for generating index/component/ilm templates
-        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
-            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
-        fi
+        # Write out file for generating index/component/ilm templates, keeping each package type separate
+        for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
+            if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
+                outfile="${package_type}_PACKAGE_COMPONENTS"
+                echo $latest_installed_package_list > "${!outfile}"
+            fi
+        done
         if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
             # Refresh installed component template list
             latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')

salt/elasticsearch/config.sls

@@ -91,6 +91,13 @@ estemplatedir:
     - group: 939
     - makedirs: True
 
+esaddontemplatedir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/addon-index
+    - user: 930
+    - group: 939
+    - makedirs: True
+
 esrolesdir:
   file.directory:
     - name: /opt/so/conf/elasticsearch/roles

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.0.8
+  version: 9.3.2
   index_clean: true
   vm:
     max_map_count: 1048576

salt/elasticsearch/enabled.sls

@@ -10,8 +10,10 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
-{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
+{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
+{%   if GLOBALS.role != 'so-heavynode' %}
+{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
+{%   endif %}
 
 include:
   - ca
@@ -117,40 +119,52 @@ escomponenttemplates:
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
     - show_changes: False
-      
-# Auto-generate templates from defaults file
+
+# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
+so_index_template_dir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/index
+    - clean: True
+    {%- if SO_MANAGED_INDICES %}
+    - require:
+      {%- for index in SO_MANAGED_INDICES %}
+      - file: so_index_template_{{index}}
+      {%- endfor %}
+    {%- endif %}
+
+# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
+#   These index templates are for the core SO datasets and are always required
 {%     for index, settings in ES_INDEX_SETTINGS.items() %}
- {%      if settings.index_template is defined %}
-es_index_template_{{index}}:
+{%       if settings.index_template is defined %}
+so_index_template_{{index}}:
   file.managed:
     - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
     - source: salt://elasticsearch/base-template.json.jinja
     - defaults:
-      TEMPLATE_CONFIG: {{ settings.index_template }}
+        TEMPLATE_CONFIG: {{ settings.index_template }}
     - template: jinja
-    - show_changes: False
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
 {%       endif %}
 {%     endfor %}
 
-{%     if TEMPLATES %}
-# Sync custom templates to /opt/so/conf/elasticsearch/templates
-{%       for TEMPLATE in TEMPLATES %}
-es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
+{%     if GLOBALS.role != "so-heavynode" %}
+# Auto-generate optional index templates for integration | input | content packages
+#   These index templates are not used by default (until user adds package to an agent policy).
+#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
+{%       for index,settings in ALL_ADDON_SETTINGS.items() %}
+{%         if settings.index_template is defined %}
+addon_index_template_{{index}}:
   file.managed:
-    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
-{%         if 'jinja' in TEMPLATE.split('.')[-1] %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
+    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+        TEMPLATE_CONFIG: {{ settings.index_template }}
     - template: jinja
-{%         else %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
-{%         endif %}
-    - user: 930
-    - group: 939
     - show_changes: False
     - onchanges_in:
-      - file: so-elasticsearch-templates-reload
+      - file: addon-elasticsearch-templates-reload
+{%         endif %}
 {%       endfor %}
 {%     endif %}
 
@@ -179,9 +193,18 @@ so-elasticsearch-templates-reload:
   file.absent:
     - name: /opt/so/state/estemplates.txt
 
+addon-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/addon_estemplates.txt
+
+# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
 so-elasticsearch-templates:
   cmd.run:
+{%-    if GLOBALS.role == "so-heavynode" %}
+    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
+{%-    else %}
     - name: /usr/sbin/so-elasticsearch-templates-load
+{%-     endif %}
     - cwd: /opt/so
     - template: jinja
     - require:

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1

@@ -10,24 +10,28 @@
   "processors": [
     {
       "set": {
+        "tag": "set_ecs_version_f5923549",
         "field": "ecs.version",
         "value": "8.17.0"
       }
     },
     {
       "set": {
+        "tag": "set_observer_vendor_ad9d35cc",
         "field": "observer.vendor",
         "value": "netgate"
       }
     },
     {
       "set": {
+        "tag": "set_observer_type_5dddf3ba",
         "field": "observer.type",
         "value": "firewall"
       }
     },
     {
       "rename": {
+        "tag": "rename_message_to_event_original_56a77271",
         "field": "message",
         "target_field": "event.original",
         "ignore_missing": true,
@@ -36,12 +40,14 @@
     },
     {
       "set": {
+        "tag": "set_event_kind_de80643c",
         "field": "event.kind",
         "value": "event"
       }
     },
     {
       "set": {
+        "tag": "set_event_timezone_4ca44cac",
         "field": "event.timezone",
         "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -49,6 +55,7 @@
     },
     {
       "grok": {
+        "tag": "grok_event_original_27d9c8c7",
         "description": "Parse syslog header",
         "field": "event.original",
         "patterns": [
@@ -72,6 +79,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
         "if": "ctx._tmp.timestamp8601 != null",
         "field": "_tmp.timestamp8601",
         "target_field": "@timestamp",
@@ -82,6 +90,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp_to_timestamp_f21e536e",
         "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
         "field": "_tmp.timestamp",
         "target_field": "@timestamp",
@@ -95,6 +104,7 @@
     },
     {
       "grok": {
+        "tag": "grok_process_name_cef3d489",
         "description": "Set Event Provider",
         "field": "process.name",
         "patterns": [
@@ -107,71 +117,83 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-firewall",
+        "tag": "pipeline_e16851a7",
+        "name": "logs-pfsense.log-1.25.1-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-openvpn",
+        "tag": "pipeline_828590b5",
+        "name": "logs-pfsense.log-1.25.1-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-ipsec",
+        "tag": "pipeline_9d37039c",
+        "name": "logs-pfsense.log-1.25.1-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-dhcp",
+        "tag": "pipeline_ad56bbca",
+        "name": "logs-pfsense.log-1.25.1-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-unbound",
+        "tag": "pipeline_dd85553d",
+        "name": "logs-pfsense.log-1.25.1-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-haproxy",
+        "tag": "pipeline_720ed255",
+        "name": "logs-pfsense.log-1.25.1-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-php-fpm",
+        "tag": "pipeline_456beba5",
+        "name": "logs-pfsense.log-1.25.1-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-squid",
+        "tag": "pipeline_a0d89375",
+        "name": "logs-pfsense.log-1.25.1-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-snort",
+        "tag": "pipeline_c2f1ed55",
+        "name": "logs-pfsense.log-1.25.1-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-suricata",
+        "tag":"pipeline_33db1c9e",
+        "name": "logs-pfsense.log-1.25.1-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
+        "tag": "drop_9d7c46f8",
         "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
       "append": {
+        "tag": "append_event_category_4780a983",
         "field": "event.category",
         "value": "network",
         "if": "ctx.network != null"
@@ -179,6 +201,7 @@
     },
     {
       "convert": {
+        "tag": "convert_source_address_to_source_ip_f5632a20",
         "field": "source.address",
         "target_field": "source.ip",
         "type": "ip",
@@ -188,6 +211,7 @@
     },
     {
       "convert": {
+        "tag": "convert_destination_address_to_destination_ip_f1388f0c",
         "field": "destination.address",
         "target_field": "destination.ip",
         "type": "ip",
@@ -197,6 +221,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_1f1d940a",
         "field": "network.type",
         "value": "ipv6",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -204,6 +229,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_69deca38",
         "field": "network.type",
         "value": "ipv4",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -211,6 +237,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_geo_da2e41b2",
         "field": "source.ip",
         "target_field": "source.geo",
         "ignore_missing": true
@@ -218,6 +245,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
         "field": "destination.ip",
         "target_field": "destination.geo",
         "ignore_missing": true
@@ -225,6 +253,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_as_28d69883",
         "ignore_missing": true,
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "source.ip",
@@ -237,6 +266,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_as_8a007787",
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "destination.ip",
         "target_field": "destination.as",
@@ -249,6 +279,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_asn_to_source_as_number_a917047d",
         "field": "source.as.asn",
         "target_field": "source.as.number",
         "ignore_missing": true
@@ -256,6 +287,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
         "field": "source.as.organization_name",
         "target_field": "source.as.organization.name",
         "ignore_missing": true
@@ -263,6 +295,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
         "field": "destination.as.asn",
         "target_field": "destination.as.number",
         "ignore_missing": true
@@ -270,6 +303,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
         "field": "destination.as.organization_name",
         "target_field": "destination.as.organization.name",
         "ignore_missing": true
@@ -277,12 +311,14 @@
     },
     {
       "community_id": {
+        "tag": "community_id_d2308e7a",
         "target_field": "network.community_id",
         "ignore_failure": true
       }
     },
     {
       "grok": {
+        "tag": "grok_observer_ingress_interface_name_968018d3",
         "field": "observer.ingress.interface.name",
         "patterns": [
           "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -293,6 +329,7 @@
     },
     {
       "set": {
+        "tag": "set_network_vlan_id_efd4d96a",
         "field": "network.vlan.id",
         "copy_from": "observer.ingress.vlan.id",
         "ignore_empty_value": true
@@ -300,6 +337,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_c1a6356b",
         "field": "related.ip",
         "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
@@ -308,6 +346,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_8121c591",
         "field": "related.ip",
         "value": "{{{source.ip}}}",
         "allow_duplicates": false,
@@ -316,6 +355,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_53b62ed8",
         "field": "related.ip",
         "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
@@ -324,6 +364,7 @@
     },
     {
       "append": {
+        "tag": "append_related_hosts_6f162628",
         "field": "related.hosts",
         "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
@@ -331,6 +372,7 @@
     },
     {
       "append": {
+        "tag": "append_related_user_c036eec2",
         "field": "related.user",
         "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
@@ -338,6 +380,7 @@
     },
     {
       "set": {
+        "tag": "set_network_direction_cb1e3125",
         "field": "network.direction",
         "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -345,6 +388,7 @@
     },
     {
       "remove": {
+        "tag": "remove_a82e20f2",
         "field": [
           "_tmp"
         ],
@@ -353,11 +397,21 @@
     },
     {
       "script": {
+        "tag": "script_a7f2c062",
         "lang": "painless",
         "description": "This script processor iterates over the whole document to remove fields with null values.",
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
+    {
+      "append": {
+        "tag": "append_preserve_original_event_on_error",
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false,
+        "if": "ctx.error?.message != null"
+      }
+    },
     {
       "pipeline": {
         "name": "global@custom",
@@ -405,7 +459,14 @@
     {
       "append": {
         "field": "error.message",
-        "value": "{{{ _ingest.on_failure_message }}}"
+        "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
+      }
+    },
+    {
+      "append": {
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false
       }
     }
   ]

salt/elasticsearch/template.map.jinja

@@ -14,15 +14,42 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
+{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %}
+{% set ALL_ADDON_SETTINGS_ORIG = {} %}
+{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %}
+{% set ALL_ADDON_SETTINGS = {} %}
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
-{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
-{%   if check_package_components.size > 1 %}
-{%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
-{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
-{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
-{%    endfor %}
-{%   endif%}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
+{#   import integration type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{%     set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{%     if check_integration_package_components.size > 1 %}
+{%       from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %}
+{%     endif %}
+{%   endif %}
+
+{#   import input type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %}
+{%     set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %}
+{%     if check_input_package_components.size > 1 %}
+{%       from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %}
+{%     endif %}
+{%   endif %}
+
+{#   import content type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %}
+{%     set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %}
+{%     if check_content_package_components.size > 1 %}
+{%       from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %}
+{%     endif %}
+{%   endif %}
+
+{%   for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %}
+{%     do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %}
+{%   endfor %}
 {% endif %}
 {# end generation of integration default index_settings #}
 
@@ -31,25 +58,33 @@
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
 {% endfor %}
 
+{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
+{%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
+{%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
+{%   endfor %}
+{% endif %}
+
 {% set ES_INDEX_SETTINGS = {} %}
-{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
-{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
+{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
+
+{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
+{% for index, settings in GLOBAL_OVERRIDES.items() %}
 
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
-{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+{%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
 
 {#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
 {#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do GLOBAL_OVERRIDES[index].pop('policy') %}
 {%     endif %}
 
 {#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
-{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
-{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
-{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%     if GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %}
+{%           do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
 {%         endif %}
 {%       endfor %}
 {%     endif %}
@@ -111,5 +146,14 @@
 {%     endfor %}
 {%   endif %}
 
-{%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
+{%   do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %}
 {% endfor %}
+{% endmacro %}
+
+{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
+{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
+
+{% set SO_MANAGED_INDICES = [] %}
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+{%   do SO_MANAGED_INDICES.append(index) %}
+{% endfor %}

salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list

@@ -6,8 +6,19 @@
 # Elastic License 2.0.
 
 . /usr/sbin/so-common
-if [ "$1" == "" ]; then
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
+
+if [[ -z "$1" ]]; then
+    if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
+        jq '[.component_templates[] | .name] | sort' <<< "$output"
+    else
+        echo "Failed to retrieve component templates from Elasticsearch."
+        exit 1
+    fi
 else
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
-fi
+    if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
+        jq <<< "$output"
+    else
+        echo "Failed to retrieve component template '$1' from Elasticsearch."
+        exit 1
+    fi
+fi

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -0,0 +1,248 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt
+ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
+ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
+SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
+ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
+SO_LOAD_FAILURES=0
+ADDON_LOAD_FAILURES=0
+SO_LOAD_FAILURES_NAMES=()
+ADDON_LOAD_FAILURES_NAMES=()
+IS_HEAVYNODE="false"
+FORCE="false"
+VERBOSE="false"
+SHOULD_EXIT_ON_FAILURE="true"
+
+# If soup is running, ignore errors
+pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+    --heavynode)
+        IS_HEAVYNODE="true"
+        ;;
+    --force)
+        FORCE="true"
+        ;;
+    --verbose)
+        VERBOSE="true"
+        ;;
+    *)
+        echo "Usage: $0 [options]"
+        echo "Options:"
+        echo "  --heavynode     Only loads index templates specific to heavynodes"
+        echo "  --force     Force reload all templates regardless of statefiles (default: false)"
+        echo "  --verbose     Enable verbose output"
+        exit 1
+        ;;
+    esac
+    shift
+done
+
+load_template() {
+    local uri="$1"
+    local file="$2"
+
+    echo "Loading template file $file"
+    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
+        echo "$output"
+
+        return 1
+
+    elif [[ "$VERBOSE" == "true" ]]; then
+        echo "$output"
+    fi
+
+}
+
+check_required_component_template_exists() {
+    local required
+    local missing
+    local file=$1
+
+    required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
+    missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
+
+    if [[ $(jq length <<<"$missing") -gt 0 ]]; then
+
+        return 1
+    fi
+}
+
+check_heavynode_compatiable_index_template() {
+    # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
+    # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
+    local -A heavynode_index_templates=(
+        ["so-import"]=1
+        ["so-syslog"]=1
+        ["so-logs-soc"]=1
+        ["so-suricata"]=1
+        ["so-suricata.alerts"]=1
+        ["so-zeek"]=1
+        ["so-strelka"]=1
+    )
+
+    local template_name="$1"
+
+    if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
+
+        return 1
+    fi
+
+}
+
+load_component_templates() {
+    local printed_name="$1"
+    local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
+    local append_mappings="${3:-"false"}"
+
+    # current state of nullglob shell option
+    shopt -q nullglob && nullglob_set=1 || nullglob_set=0
+
+    shopt -s nullglob
+    echo -e "\nLoading $printed_name component templates...\n"
+    for component in "$pattern"/*.json; do
+        tmpl_name=$(basename "${component%.json}")
+
+        if [[ "$append_mappings" == "true" ]]; then
+            # avoid duplicating "-mappings" if it already exists in the component template filename
+            tmpl_name="${tmpl_name%-mappings}-mappings"
+        fi
+
+        if ! load_template "_component_template/${tmpl_name}" "$component"; then
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$component")
+        fi
+    done
+
+    # restore nullglob shell option if needed
+    if [[ $nullglob_set -eq 1 ]]; then
+        shopt -u nullglob
+    fi
+}
+
+check_elasticsearch_responsive() {
+    # Cannot load templates if Elasticsearch is not responding.
+    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
+    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
+    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
+        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
+}
+
+if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]]; then
+    check_elasticsearch_responsive
+
+    if [[ "$IS_HEAVYNODE" == "false" ]]; then
+        # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
+        fleet_check="logs-endpoint.alerts@package"
+        if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
+            # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run.
+            echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
+            # Fleet Server is required because some SO index templates depend on components installed via
+            #  specific integrations eg Elastic Defend. These are components that we do not manually create / manage
+            #  via /opt/so/saltstack/salt/elasticsearch/templates/component/
+
+            exit 0
+        fi
+    fi
+
+    # load_component_templates "Name" "directory" "append '-mappings'?"
+    load_component_templates "ECS" "ecs" "true"
+    load_component_templates "Elastic Agent" "elastic-agent"
+    load_component_templates "Security Onion" "so"
+
+    component_templates=$(so-elasticsearch-component-templates-list)
+    echo -e "Loading Security Onion index templates...\n"
+    for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
+        tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
+
+        if [[ "$IS_HEAVYNODE" == "true" ]]; then
+            # TODO: Better way to load only heavynode specific templates
+            if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
+                if [[ "$VERBOSE" == "true" ]]; then
+                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
+                fi
+
+                continue
+            fi
+        fi
+
+        if check_required_component_template_exists "$so_idx_tmpl"; then
+            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
+                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+            fi
+        else
+            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+
+            continue
+        fi
+    done
+
+    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
+        echo "All Security Onion core templates loaded successfully."
+
+        touch "$SO_STATEFILE_SUCCESS"
+    else
+        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
+        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
+            echo "  - $failed_template"
+        done
+        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
+            fail "Failed to load all Security Onion core templates successfully."
+        fi
+    fi
+else
+
+    echo "Security Onion core templates already loaded"
+fi
+
+# Start loading addon templates
+if [[ (-f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || "$FORCE" == "true" ]]; then
+
+    check_elasticsearch_responsive
+
+    echo -e "\nLoading addon integration index templates...\n"
+    component_templates=$(so-elasticsearch-component-templates-list)
+
+    for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do
+        tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
+
+        if check_required_component_template_exists "$addon_idx_tmpl"; then
+            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
+                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+            fi
+        else
+            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
+            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+
+            continue
+        fi
+    done
+
+    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
+        echo "All addon integration templates loaded successfully."
+
+        touch "$ADDON_STATEFILE_SUCCESS"
+    else
+        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
+        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
+            echo "  - $failed_template"
+        done
+        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
+            fail "Failed to load all addon integration templates successfully."
+        fi
+    fi
+
+fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -1,165 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-{%  from 'vars/globals.map.jinja' import GLOBALS %}
-
-STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
-STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
-
-if [[ -f $STATE_FILE_INITIAL ]]; then
-  # The initial template load has already run. As this is a subsequent load, all dependencies should 
-  # already be satisified. Therefore, immediately exit/abort this script upon any template load failure
-  # since this is an unrecoverable failure.
-  should_exit_on_failure=1
-else
-  # This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
-  # Therefore load as many templates as possible at this time and if an error occurs proceed to the next 
-  # template. But if at least one template fails to load do not mark the templates as having been loaded.
-  # This will allow the next load to resume the load of the templates that failed to load initially.
-  should_exit_on_failure=0
-  echo "This is the initial template load"
-fi
-
-# If soup is running, ignore errors
-pgrep soup > /dev/null && should_exit_on_failure=0
-
-load_failures=0
-
-load_template() {
-  uri=$1
-  file=$2
-
-  echo "Loading template file $i" 
-  if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
-    if [[ $should_exit_on_failure -eq 1 ]]; then
-      fail "Could not load template file: $file"
-    else
-      load_failures=$((load_failures+1))
-      echo "Incremented load failure counter: $load_failures"
-    fi
-  fi
-}
-
-if [ ! -f $STATE_FILE_SUCCESS ]; then
-  echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
-
-  . /usr/sbin/so-common
-
-  {% if GLOBALS.role != 'so-heavynode' %}
-  if [ -f /usr/sbin/so-elastic-fleet-common ]; then
-    . /usr/sbin/so-elastic-fleet-common
-  fi
-  {% endif %}
-
-  default_conf_dir=/opt/so/conf
-
-  # Define a default directory to load pipelines from
-  ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
-
-  {% if GLOBALS.role == 'so-heavynode' %}
-  file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
-  {% else %}
-  file="/usr/sbin/so-elastic-fleet-common"
-  {% endif %}
-
-  if [ -f "$file" ]; then
-    # Wait for ElasticSearch to initialize
-    echo -n "Waiting for ElasticSearch..."
-    retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-    {% if GLOBALS.role != 'so-heavynode' %}
-    TEMPLATE="logs-endpoint.alerts@package"
-    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
-    if [ "$INSTALLED" != "$TEMPLATE" ]; then
-      echo
-      echo "Packages not yet installed."
-      echo
-      exit 0
-    fi
-    {% endif %}
-
-    touch $STATE_FILE_INITIAL
-
-    cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
-
-    echo "Loading ECS component templates..."
-    for i in *; do 
-      TEMPLATE=$(echo $i | cut -d '.' -f1) 
-      load_template "_component_template/${TEMPLATE}-mappings" "$i"
-    done
-    echo
-
-    cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
-
-    echo "Loading Elastic Agent component templates..."
-    {% if GLOBALS.role == 'so-heavynode' %}
-    component_pattern="so-*"
-    {% else %}
-    component_pattern="*"
-    {% endif %}
-    for i in $component_pattern; do 
-      TEMPLATE=${i::-5} 
-      load_template "_component_template/$TEMPLATE" "$i"
-    done
-    echo
-	    
-    # Load SO-specific component templates
-    cd ${ELASTICSEARCH_TEMPLATES}/component/so
-
-    echo "Loading Security Onion component templates..."
-    for i in *; do 
-      TEMPLATE=$(echo $i | cut -d '.' -f1); 
-      load_template "_component_template/$TEMPLATE" "$i"
-    done
-    echo
-
-    # Load SO index templates
-    cd ${ELASTICSEARCH_TEMPLATES}/index
-
-    echo "Loading Security Onion index templates..."
-    shopt -s extglob
-    {% if GLOBALS.role == 'so-heavynode' %}
-    pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
-    {% else %}
-    pattern="*"
-    {% endif %}
-    # Index templates will be skipped if the following conditions are met:
-    # 1. The template is part of the "so-logs-" template group
-    # 2. The template name does not correlate to at least one existing component template
-    # In this situation, the script will treat the skipped template as a temporary failure
-    # and allow the templates to be loaded again on the next run or highstate, whichever 
-    # comes first.
-    COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
-    for i in $pattern; do
-      TEMPLATE=${i::-14}
-      COMPONENT_PATTERN=${TEMPLATE:3}
-      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
-        load_failures=$((load_failures+1))
-        echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
-      else
-        load_template "_index_template/$TEMPLATE" "$i"
-      fi
-    done
-  else
-    {% if GLOBALS.role == 'so-heavynode' %}
-    echo "Common template does not exist. Exiting..."
-    {% else %}
-    echo "Elastic Fleet not configured. Exiting..."
-    {% endif %}
-    exit 0
-  fi
-  
-  cd - >/dev/null
-
-  if [[ $load_failures -eq 0 ]]; then
-    echo "All templates loaded successfully"
-    touch $STATE_FILE_SUCCESS
-  else
-    echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
-  fi
-else
-  echo "Templates already loaded"
-fi

salt/firewall/containers.map.jinja

@@ -11,7 +11,6 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
-     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -35,7 +34,6 @@
      'so-hydra',
      'so-logstash',
      'so-nginx',
-     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -79,7 +77,6 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
-     'so-postgres',
      'so-soc'
 ] %}
 

salt/firewall/defaults.yaml

@@ -98,10 +98,6 @@ firewall:
       tcp:
         - 8086
       udp: []
-    postgres:
-      tcp:
-        - 5432
-      udp: []
     kafka_controller:
       tcp:
         - 9093
@@ -197,7 +193,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - localrules
@@ -384,7 +379,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -596,7 +590,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -806,7 +799,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -1019,7 +1011,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/postgres/auth.sls

@@ -1,35 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-  {% set DIGITS = "1234567890" %}
-  {% set LOWERCASE = "qwertyuiopasdfghjklzxcvbnm" %}
-  {% set UPPERCASE = "QWERTYUIOPASDFGHJKLZXCVBNM" %}
-  {% set SYMBOLS = "~!@#^&*()-_=+[]|;:,.<>?" %}
-  {% set CHARS = DIGITS~LOWERCASE~UPPERCASE~SYMBOLS %}
-  {% set so_postgres_user_pass = salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', salt['random.get_str'](72, chars=CHARS)) %}
-
-postgres_auth_pillar:
-  file.managed:
-    - name: /opt/so/saltstack/local/pillar/postgres/auth.sls
-    - mode: 640
-    - reload_pillar: True
-    - contents: |
-        postgres:
-          auth:
-            users:
-              so_postgres_user:
-                user: so_postgres
-                pass: "{{ so_postgres_user_pass }}"
-    - show_changes: False
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/postgres/config.sls

@@ -1,71 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'postgres/map.jinja' import PGMERGED %}
-
-# Postgres Setup
-postgresconfdir:
-  file.directory:
-    - name: /opt/so/conf/postgres
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-postgresdatadir:
-  file.directory:
-    - name: /nsm/postgres
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-postgreslogdir:
-  file.directory:
-    - name: /opt/so/log/postgres
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-postgresinitdir:
-  file.directory:
-    - name: /opt/so/conf/postgres/init
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-postgresinitusers:
-  file.managed:
-    - name: /opt/so/conf/postgres/init/init-users.sh
-    - source: salt://postgres/files/init-users.sh
-    - user: 939
-    - group: 939
-    - mode: 755
-
-postgresconf:
-  file.managed:
-    - name: /opt/so/conf/postgres/postgresql.conf
-    - source: salt://postgres/files/postgresql.conf.jinja
-    - user: 939
-    - group: 939
-    - template: jinja
-    - defaults:
-        PGMERGED: {{ PGMERGED }}
-
-postgres_sbin:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://postgres/tools/sbin
-    - user: 939
-    - group: 939
-    - file_mode: 755
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/postgres/defaults.yaml

@@ -1,14 +0,0 @@
-postgres:
-  enabled: True
-  config:
-    listen_addresses: '*'
-    port: 5432
-    max_connections: 100
-    shared_buffers: 256MB
-    ssl: 'on'
-    ssl_cert_file: '/conf/postgres.crt'
-    ssl_key_file: '/conf/postgres.key'
-    ssl_ca_file: '/conf/ca.crt'
-    log_destination: 'stderr'
-    logging_collector: 'off'
-    log_min_messages: 'warning'

salt/postgres/disabled.sls

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-include:
-  - postgres.sostatus
-
-so-postgres:
-  docker_container.absent:
-    - force: True
-
-so-postgres_so-status.disabled:
-  file.comment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-postgres$
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/postgres/enabled.sls

@@ -1,88 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
-{%   set PASSWORD = salt['pillar.get']('secrets:postgres_pass') %}
-{%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
-{%   set SO_POSTGRES_PASS = salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', '') %}
-
-include:
-  - postgres.auth
-  - postgres.ssl
-  - postgres.config
-  - postgres.sostatus
-
-so-postgres:
-  docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }}
-    - hostname: so-postgres
-    - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-postgres'].ip }}
-    - port_bindings:
-      {% for BINDING in DOCKERMERGED.containers['so-postgres'].port_bindings %}
-      - {{ BINDING }}
-      {% endfor %}
-    - environment:
-      - POSTGRES_DB=securityonion
-      - POSTGRES_PASSWORD={{ PASSWORD }}
-      - SO_POSTGRES_USER={{ SO_POSTGRES_USER }}
-      - SO_POSTGRES_PASS={{ SO_POSTGRES_PASS }}
-      {% if DOCKERMERGED.containers['so-postgres'].extra_env %}
-        {% for XTRAENV in DOCKERMERGED.containers['so-postgres'].extra_env %}
-      - {{ XTRAENV }}
-        {% endfor %}
-      {% endif %}
-    - binds:
-      - /opt/so/log/postgres/:/log:rw
-      - /nsm/postgres:/var/lib/postgresql/data:rw
-      - /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
-      - /opt/so/conf/postgres/init/init-users.sh:/docker-entrypoint-initdb.d/init-users.sh:ro
-      - /etc/pki/postgres.crt:/conf/postgres.crt:ro
-      - /etc/pki/postgres.key:/conf/postgres.key:ro
-      - /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro
-      {% if DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %}
-        {% for BIND in DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %}
-      - {{ BIND }}
-        {% endfor %}
-      {% endif %}
-    {% if DOCKERMERGED.containers['so-postgres'].extra_hosts %}
-    - extra_hosts:
-      {% for XTRAHOST in DOCKERMERGED.containers['so-postgres'].extra_hosts %}
-      - {{ XTRAHOST }}
-      {% endfor %}
-    {% endif %}
-    {% if DOCKERMERGED.containers['so-postgres'].ulimits %}
-    - ulimits:
-    {%   for ULIMIT in DOCKERMERGED.containers['so-postgres'].ulimits %}
-      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
-    {%   endfor %}
-    {% endif %}
-    - watch:
-      - file: postgresconf
-      - file: postgresinitusers
-      - x509: postgres_crt
-      - x509: postgres_key
-    - require:
-      - file: postgresconf
-      - file: postgresinitusers
-      - x509: postgres_crt
-      - x509: postgres_key
-
-delete_so-postgres_so-status.disabled:
-  file.uncomment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-postgres$
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/postgres/files/init-users.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-set -e
-
-# Create or update application user for SOC platform access
-# This script runs on first database initialization via docker-entrypoint-initdb.d
-# The password is properly escaped to handle special characters
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    DO \$\$
-    BEGIN
-        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
-            EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        ELSE
-            EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        END IF;
-    END
-    \$\$;
-    GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
-EOSQL

salt/postgres/files/postgresql.conf.jinja

@@ -1,8 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-
-{% for key, value in PGMERGED.config.items() %}
-{{ key }} = '{{ value }}'
-{% endfor %}

salt/postgres/init.sls

@@ -1,13 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'postgres/map.jinja' import PGMERGED %}
-
-include:
-{% if PGMERGED.enabled %}
-  - postgres.enabled
-{% else %}
-  - postgres.disabled
-{% endif %}

salt/postgres/map.jinja

@@ -1,7 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-
-{% import_yaml 'postgres/defaults.yaml' as PGDEFAULTS %}
-{% set PGMERGED = salt['pillar.get']('postgres', PGDEFAULTS.postgres, merge=True) %}

salt/postgres/sostatus.sls

@@ -1,21 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-append_so-postgres_so-status.conf:
-  file.append:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - text: so-postgres
-    - unless: grep -q so-postgres /opt/so/conf/so-status/so-status.conf
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/postgres/ssl.sls

@@ -1,54 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'ca/map.jinja' import CA %}
-
-postgres_key:
-  x509.private_key_managed:
-    - name: /etc/pki/postgres.key
-    - keysize: 4096
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/postgres.key') -%}
-    - prereq:
-      - x509: /etc/pki/postgres.crt
-    {%- endif %}
-    - retry:
-        attempts: 5
-        interval: 30
-
-postgres_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/postgres.crt
-    - ca_server: {{ CA.server }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - signing_policy: postgres
-    - private_key: /etc/pki/postgres.key
-    - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 7
-    - days_valid: 820
-    - backup: True
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-
-postgresKeyperms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/postgres.key
-    - mode: 640
-    - group: 939
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/postgres/tools/sbin/so-postgres-manage

@@ -1,80 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-usage() {
-    echo "Usage: $0 <operation> [args]"
-    echo ""
-    echo "Supported Operations:"
-    echo "  sql          Execute a SQL command, requires: <sql>"
-    echo "  sqlfile      Execute a SQL file, requires: <path>"
-    echo "  shell        Open an interactive psql shell"
-    echo "  dblist       List databases"
-    echo "  userlist     List database roles"
-    echo ""
-    exit 1
-}
-
-if [ $# -lt 1 ]; then
-  usage
-fi
-
-# Check for prerequisites
-if [ "$(id -u)" -ne 0 ]; then
-    echo "This script must be run using sudo!"
-    exit 1
-fi
-
-COMMAND=$(basename $0)
-OP=$1
-shift
-
-set -eo pipefail
-
-log() {
-  echo -e "$(date) | $COMMAND | $@" >&2
-}
-
-so_psql() {
-  docker exec so-postgres psql -U postgres -d securityonion "$@"
-}
-
-case "$OP" in
-
-  sql)
-    [ $# -lt 1 ] && usage
-    so_psql -c "$1"
-    ;;
-
-  sqlfile)
-    [ $# -ne 1 ] && usage
-    if [ ! -f "$1" ]; then
-      log "File not found: $1"
-      exit 1
-    fi
-    docker cp "$1" so-postgres:/tmp/sqlfile.sql
-    docker exec so-postgres psql -U postgres -d securityonion -f /tmp/sqlfile.sql
-    docker exec so-postgres rm -f /tmp/sqlfile.sql
-    ;;
-
-  shell)
-    docker exec -it so-postgres psql -U postgres -d securityonion
-    ;;
-
-  dblist)
-    so_psql -c "\l"
-    ;;
-
-  userlist)
-    so_psql -c "\du"
-    ;;
-
-  *)
-    usage
-    ;;
-esac

salt/postgres/tools/sbin/so-postgres-restart

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-restart postgres $1

salt/postgres/tools/sbin/so-postgres-start

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start postgres $1

salt/postgres/tools/sbin/so-postgres-stop

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop postgres $1

salt/soc/defaults.map.jinja

@@ -24,11 +24,6 @@
 
 {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
-{% if GLOBALS.postgres is defined and GLOBALS.postgres.auth is defined %}
-{%   set PG_ADMIN_PASS = salt['pillar.get']('secrets:postgres_pass', '') %}
-{% do SOCDEFAULTS.soc.config.server.modules.update({'postgres': {'hostUrl': GLOBALS.manager_ip, 'port': 5432, 'username': GLOBALS.postgres.auth.users.so_postgres_user.user, 'password': GLOBALS.postgres.auth.users.so_postgres_user.pass, 'adminUser': 'postgres', 'adminPassword': PG_ADMIN_PASS, 'dbname': 'securityonion', 'sslMode': 'require', 'assistantEnabled': true, 'esHostUrl': 'https://' ~ GLOBALS.manager_ip ~ ':9200', 'esUsername': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'esPassword': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}}) %}
-{% endif %}
-
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
 {% for tool in SOCDEFAULTS.soc.config.server.client.tools %}

salt/top.sls

@@ -68,7 +68,6 @@ base:
     - backup.config_backup
     - nginx
     - influxdb
-    - postgres
     - soc
     - kratos
     - hydra
@@ -96,7 +95,6 @@ base:
     - backup.config_backup
     - nginx
     - influxdb
-    - postgres
     - soc
     - kratos
     - hydra
@@ -125,7 +123,6 @@ base:
     - registry
     - nginx
     - influxdb
-    - postgres
     - strelka.manager
     - soc
     - kratos
@@ -156,7 +153,6 @@ base:
     - registry
     - nginx
     - influxdb
-    - postgres
     - strelka.manager
     - soc
     - kratos
@@ -185,7 +181,6 @@ base:
     - manager
     - nginx
     - influxdb
-    - postgres
     - strelka.manager
     - soc
     - kratos

salt/vars/eval.map.jinja

@@ -1,5 +1,4 @@
 {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
-{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %}
 {% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %}
 
 {% set ROLE_GLOBALS = {} %}
@@ -7,7 +6,6 @@
 {% set EVAL_GLOBALS =
    [
      ELASTICSEARCH_GLOBALS,
-     POSTGRES_GLOBALS,
      SENSOR_GLOBALS
    ]
 %}

salt/vars/import.map.jinja

@@ -1,5 +1,4 @@
 {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
-{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %}
 {% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %}
 
 {% set ROLE_GLOBALS = {} %}
@@ -7,7 +6,6 @@
 {% set IMPORT_GLOBALS =
    [
      ELASTICSEARCH_GLOBALS,
-     POSTGRES_GLOBALS,
      SENSOR_GLOBALS
    ]
 %}

salt/vars/manager.map.jinja

@@ -1,14 +1,12 @@
 {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
 {% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %}
-{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %}
 
 {% set ROLE_GLOBALS = {} %}
 
 {% set MANAGER_GLOBALS =
    [
      ELASTICSEARCH_GLOBALS,
-     LOGSTASH_GLOBALS,
-     POSTGRES_GLOBALS
+     LOGSTASH_GLOBALS
    ]
 %}
 

salt/vars/managersearch.map.jinja

@@ -1,14 +1,12 @@
 {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
 {% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %}
-{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %}
 
 {% set ROLE_GLOBALS = {} %}
 
 {% set MANAGERSEARCH_GLOBALS =
    [
      ELASTICSEARCH_GLOBALS,
-     LOGSTASH_GLOBALS,
-     POSTGRES_GLOBALS
+     LOGSTASH_GLOBALS
    ]
 %}
 

salt/vars/postgres.map.jinja

@@ -1,16 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-
-{% import 'vars/init.map.jinja' as INIT %}
-
-{%
-  set POSTGRES_GLOBALS = {
-    'postgres': {}
-  }
-%}
-
-{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-{%   do POSTGRES_GLOBALS.postgres.update({'auth': INIT.PILLAR.postgres.auth}) %}
-{% endif %}

salt/vars/standalone.map.jinja

@@ -1,6 +1,5 @@
 {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
 {% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %}
-{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %}
 {% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %}
 
 {% set ROLE_GLOBALS = {} %}
@@ -9,7 +8,6 @@
    [
      ELASTICSEARCH_GLOBALS,
      LOGSTASH_GLOBALS,
-     POSTGRES_GLOBALS,
      SENSOR_GLOBALS
    ]
 %}

salt/zeek/soc_zeek.yaml

@@ -5,7 +5,7 @@ zeek:
     helpLink: zeek
   ja4plus:
     enabled:
-      description: "Enables JA4+ fingerprinting (JA4S, JA4D, JA4H, JA4L, JA4SSH, JA4T, JA4TS, JA4X). By enabling this, you agree to the terms of the JA4+ license  [https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE](https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE)."
+      description: "Enables JA4+ fingerprinting (JA4S, JA4D, JA4H, JA4L, JA4SSH, JA4T, JA4TS, JA4X). By enabling this, you agree to the terms of the JA4+ license  [https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4](https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4)."
       forcedType: bool
       helpLink: zeek
       advanced: False

setup/so-functions

@@ -821,7 +821,6 @@ create_manager_pillars() {
 	soc_pillar
 	idh_pillar
 	influxdb_pillar
-	postgres_pillar
 	logrotate_pillar
 	patch_pillar
 	nginx_pillar
@@ -1054,7 +1053,6 @@ generate_passwords(){
   HYDRAKEY=$(get_random_value)
   HYDRASALT=$(get_random_value)
   REDISPASS=$(get_random_value)
-  POSTGRESPASS=$(get_random_value)
   SOCSRVKEY=$(get_random_value 64)
   IMPORTPASS=$(get_random_value)
 }
@@ -1357,12 +1355,6 @@ influxdb_pillar() {
 		"  token: $INFLUXTOKEN" > $local_salt_dir/pillar/influxdb/token.sls
 }
 
-postgres_pillar() {
-	title "Create the postgres pillar file"
-	touch $adv_postgres_pillar_file
-	touch $postgres_pillar_file
-}
-
 make_some_dirs() {
 	mkdir -p /nsm
 	mkdir -p "$default_salt_dir"
@@ -1372,7 +1364,7 @@ make_some_dirs() {
 	mkdir -p $local_salt_dir/salt/firewall/portgroups
 	mkdir -p $local_salt_dir/salt/firewall/ports
 
-	for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb postgres strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do
+	for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do
 	mkdir -p $local_salt_dir/pillar/$THEDIR
 	touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
 	touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls
@@ -1840,8 +1832,7 @@ secrets_pillar(){
 	printf '%s\n'\
 		"secrets:"\
 		"  import_pass: $IMPORTPASS"\
-		"  influx_pass: $INFLUXPASS"\
-		"  postgres_pass: $POSTGRESPASS" > $local_salt_dir/pillar/secrets.sls
+		"  influx_pass: $INFLUXPASS" > $local_salt_dir/pillar/secrets.sls
   fi
 }
 

setup/so-variables

@@ -202,12 +202,6 @@ export influxdb_pillar_file
 adv_influxdb_pillar_file="$local_salt_dir/pillar/influxdb/adv_influxdb.sls"
 export adv_influxdb_pillar_file
 
-postgres_pillar_file="$local_salt_dir/pillar/postgres/soc_postgres.sls"
-export postgres_pillar_file
-
-adv_postgres_pillar_file="$local_salt_dir/pillar/postgres/adv_postgres.sls"
-export adv_postgres_pillar_file
-
 logrotate_pillar_file="$local_salt_dir/pillar/logrotate/soc_logrotate.sls"
 export logrotate_pillar_file