merge with 2.4.120, fix merge conflicts

salt/soc/config.sls

@@ -90,7 +90,7 @@ filedetectionsbackup:
 
 crondetectionsruntime:
   cron.present:
-    - name: /usr/sbin/so-detections-runtime-status cron 
+    - name: /usr/sbin/so-detections-runtime-status cron
     - identifier: detections-runtime-status
     - user: root
     - minute: '*/10'
@@ -190,6 +190,14 @@ socsigmarepo:
     - group: 939
     - mode: 775
 
+socsensoronirepos:
+  file.directory:
+    - name: /opt/so/conf/soc/ai_summary_repos
+    - user: 939
+    - group: 939
+    - mode: 775
+    - makedirs: True
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/soc/defaults.yaml

@@ -1312,6 +1312,10 @@ soc:
         kratos:
           hostUrl:
         elastalertengine:
+          aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources
+          aiRepoBranch: generated-summaries-published
+          aiRepoPath: /opt/sensoroni/ai_summary_repos
+          showAiSummaries: true
           autoUpdateEnabled: true
           autoEnabledSigmaRules:
             default:
@@ -1391,6 +1395,10 @@ soc:
           userFiles:
             - rbac/users_roles
         strelkaengine:
+          aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources
+          aiRepoBranch: generated-summaries-published
+          aiRepoPath: /opt/sensoroni/ai_summary_repos
+          showAiSummaries: true
           autoEnabledYaraRules:
             - securityonion-yara
           autoUpdateEnabled: true
@@ -1412,6 +1420,10 @@ soc:
           stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state
           integrityCheckFrequencySeconds: 1200
         suricataengine:
+          aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources
+          aiRepoBranch: generated-summaries-published
+          aiRepoPath: /opt/sensoroni/ai_summary_repos
+          showAiSummaries: true
           autoUpdateEnabled: true
           communityRulesImportFrequencySeconds: 86400
           communityRulesImportErrorSeconds: 300
@@ -2232,7 +2244,7 @@ soc:
               description: Show all custom detections
             - name: "All Detections - Enabled"
               query: "so_detection.isEnabled:true | groupby so_detection.language | groupby so_detection.ruleset so_detection.severity"
-              description: Show all enalbed Detections
+              description: Show all enabled Detections
             - name: "All Detections - Disabled"
               query: "so_detection.isEnabled:false | groupby so_detection.language | groupby so_detection.ruleset so_detection.severity"
               description: Show all disabled Detections
@@ -2251,7 +2263,18 @@ soc:
             - name: "Detections with Overrides"
               query: "_exists_:so_detection.overrides | groupby so_detection.language | groupby so_detection.ruleset so_detection.isEnabled"
               description: Show Detections that have Overrides
+          detectionEngineStatusQueries: |
+            suricata:
+              default: 'tags:so-soc AND suricata | groupby log.level | groupby event.action | groupby soc.fields.error'
+              IntegrityFailure: 'event.action: "integrity check failed" AND soc.fields.detectionEngine:"suricata" | table event.dataset soc.fields.deployedButNotEnabledCount soc.fields.enabledButNotDeployedCount soc.fields.syncId'
+            elastalert:
+              default: 'tags:so-soc AND elastalert | groupby log.level | groupby event.action | groupby soc.fields.error'
+              IntegrityFailure: 'event.action: "integrity check failed" AND soc.fields.detectionEngine:"elastalert" | table event.dataset soc.fields.deployedButNotEnabledCount soc.fields.enabledButNotDeployedCount soc.fields.syncId'
+            strelka:
+              default: 'tags:so-soc AND strelka | groupby log.level | groupby event.action | groupby soc.fields.error'
+              IntegrityFailure: 'event.action: "integrity check failed" AND soc.fields.detectionEngine:"strelka" | table event.dataset soc.fields.deployedButNotEnabledCount soc.fields.enabledButNotDeployedCount soc.fields.syncId'
         detection:
+          showUnreviewedAiSummaries: false
           presets:
             severity:
                 customEnabled: false

salt/soc/enabled.sls

@@ -33,6 +33,7 @@ so-soc:
       - /nsm/soc/uploads:/nsm/soc/uploads:rw
       - /opt/so/log/soc/:/opt/sensoroni/logs/:rw
       - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro
+      - /opt/so/conf/soc/ai_summary_repos:/opt/sensoroni/ai_summary_repos:rw
 {% if SOCMERGED.telemetryEnabled and not GLOBALS.airgap %}
       - /opt/so/conf/soc/analytics.js:/opt/sensoroni/html/js/analytics.js:ro
 {% endif %}

salt/soc/soc_soc.yaml

@@ -1,6 +1,6 @@
 soc:
   enabled:
-    description: You can enable or disable SOC.
+    description: Enables or disables SOC. WARNING - Disabling this setting is unsupported and will cause the grid to malfunction. Re-enabling this setting is a manual effort via SSH.
     advanced: True
   telemetryEnabled:
     title: SOC Telemetry
@@ -87,13 +87,134 @@ soc:
         global: True
       modules:
         elastalertengine:
-          additionalAlerters:
-            title: Additional Alerters
-            description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter' to find this related setting. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+          aiRepoUrl:
+            description: URL to the AI repository. This is used to pull in AI models for use in ElastAlert rules.
             global: True
-            helpLink: sigma.html
+            advanced: True
+          aiRepoBranch:
+            description: The branch to pull from the AI repository. Leaving this blank will pull the default branch.
+            global: True
+            advanced: True
+          aiRepoPath:
+            description: Path to the AI repository. This is used to pull in AI models for use in ElastAlert rules.
+            global: True
+            advanced: True
+          showAiSummaries:
+            description: Show AI summaries for ElastAlert rules.
+            global: True
+          additionalAlerters:
+            title: "Notifications: Sev 0/Default Alerters"
+            description: "Specify default alerters to enable for outbound notifications. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+            global: True
+            helpLink: notifications.html
             forcedType: "[]string"
             multiline: True
+          additionalSev0AlertersParams:
+            title: "Notifications: Sev 0/Default Parameters"
+            description: Optional configuration parameters for default alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+            global: True
+            multiline: True
+            syntax: yaml
+            helpLink: notifications.html
+            forcedType: string
+            jinjaEscaped: True
+          additionalSev1Alerters:
+            title: "Notifications: Sev 1/Informational Alerters"
+            description: "Specify specific alerters to use when alerting at the info severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+            global: True
+            helpLink: notifications.html
+            forcedType: "[]string"
+            multiline: True
+          additionalSev1AlertersParams:
+            title: "Notifications: Sev 1/Informational Parameters"
+            description: Optional configuration parameters for informational severity alerters. Info level is less severe than 'Low Severity'. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+            global: True
+            multiline: True
+            syntax: yaml
+            helpLink: notifications.html
+            forcedType: string
+            jinjaEscaped: True
+          additionalSev2Alerters:
+            title: "Notifications: Sev 2/Low Alerters"
+            description: "Specify specific alerters to use when alerting at the low severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+            global: True
+            helpLink: notifications.html
+            forcedType: "[]string"
+            multiline: True
+          additionalSev2AlertersParams:
+            title: "Notifications: Sev 2/Low Parameters"
+            description: Optional configuration parameters for low severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+            global: True
+            multiline: True
+            syntax: yaml
+            helpLink: notifications.html
+            forcedType: string
+            jinjaEscaped: True
+          additionalSev3Alerters:
+            title: "Notifications: Sev 3/Medium Alerters"
+            description: "Specify specific alerters to use when alerting at the medium severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+            global: True
+            helpLink: notifications.html
+            forcedType: "[]string"
+            multiline: True
+          additionalSev3AlertersParams:
+            title: "Notifications: Sev 3/Medium Parameters"
+            description: Optional configuration parameters for medium severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+            global: True
+            multiline: True
+            syntax: yaml
+            helpLink: notifications.html
+            forcedType: string
+            jinjaEscaped: True
+          additionalSev4Alerters:
+            title: "Notifications: Sev 4/High Alerters"
+            description: "Specify specific alerters to use when alerting at the high severity level or critical severity level. These alerters will be used unless overridden by critical severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+            global: True
+            helpLink: notifications.html
+            forcedType: "[]string"
+            multiline: True
+          additionalSev4AlertersParams:
+            title: "Notifications: Sev 4/High Parameters"
+            description: Optional configuration parameters for high severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+            global: True
+            multiline: True
+            syntax: yaml
+            helpLink: notifications.html
+            forcedType: string
+            jinjaEscaped: True
+          additionalSev5Alerters:
+            title: "Notifications: Sev 5/Critical Alerters"
+            description: "Specify specific alerters to use when alerting at the critical severity level. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+            global: True
+            helpLink: notifications.html
+            forcedType: "[]string"
+            multiline: True
+          additionalSev5AlertersParams:
+            title: "Notifications: Sev 5/Critical Parameters"
+            description: Optional configuration parameters for critical severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+            global: True
+            multiline: True
+            syntax: yaml
+            helpLink: notifications.html
+            forcedType: string
+            jinjaEscaped: True
+          additionalUserDefinedNotifications:
+            customAlerters:
+              description: "Specify custom notification alerters to use when the Sigma rule contains the following tag: so.alerters.customAlerters. This setting can be duplicated to create new custom alerter configurations. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+              global: True
+              helpLink: notifications.html
+              forcedType: "[]string"
+              duplicates: True
+              multiline: True
+            customAlertersParams:
+              description: "Optional configuration parameters for custom notification alerters, used when the Sigma rule contains the following tag: so.params.customAlertersParams. This setting can be duplicated to create new custom alerter configurations. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
+              global: True
+              multiline: True
+              syntax: yaml
+              helpLink: notifications.html
+              duplicates: True
+              forcedType: string
+              jinjaEscaped: True
           autoEnabledSigmaRules:
             default: &autoEnabledSigmaRules
               description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.'
@@ -193,6 +314,21 @@ soc:
             advanced: True
             forcedType: int
         strelkaengine:
+          aiRepoUrl:
+            description: URL to the AI repository. This is used to pull in AI models for use in Strelka rules.
+            global: True
+            advanced: True
+          aiRepoBranch:
+            description: The branch to pull from the AI repository. Leaving this blank will pull the default branch.
+            global: True
+            advanced: True
+          aiRepoPath:
+            description: Path to the AI repository. This is used to pull in AI models for use in Strelka rules.
+            global: True
+            advanced: True
+          showAiSummaries:
+            description: Show AI summaries for Strelka rules.
+            global: True
           autoEnabledYaraRules:
             description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara'
             global: True
@@ -216,6 +352,21 @@ soc:
               helpLink: yara.html
             airgap: *serulesRepos
         suricataengine:
+          aiRepoUrl:
+            description: URL to the AI repository. This is used to pull in AI models for use in Suricata rules.
+            global: True
+            advanced: True
+          aiRepoBranch:
+            description: The branch to pull from the AI repository. Leaving this blank will pull the default branch.
+            global: True
+            advanced: True
+          aiRepoPath:
+            description: Path to the AI repository. This is used to pull in AI models for use in Suricata rules.
+            global: True
+            advanced: True
+          showAiSummaries:
+            description: Show AI summaries for Suricata rules.
+            global: True
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new Suricata rules (in seconds).'
             global: True
@@ -310,8 +461,18 @@ soc:
         alerts: *appSettings
         cases: *appSettings
         dashboards: *appSettings
-        detections: *appSettings
+        detections:
+          <<: *appSettings
+          detectionEngineStatusQueries:
+            description: Queries mapped to the detection engine statuses. Acceptable statuses are "Migrating", "Importing", "MigrationFailure", "IntegrityFailure", "SyncFailure", "ImportPending", "Syncing", and "Healthy" and will fallback to a "default" entry if specified.
+            global: True
+            syntax: yaml
+            multiline: True
+            forcedType: "string"
         detection:
+          showUnreviewedAiSummaries:
+            description: Show AI summaries in detections even if they have not yet been reviewed by a human.
+            global: True
           templateDetections:
             suricata:
               description: The template used when creating a new Suricata detection. [publicId] will be replaced with an unused Public Id.