Merge pull request #15742 from Security-Onion-Solutions/mwright/ai-query-length

Assistant: charsPerTokenEstimate

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -10,6 +10,7 @@ body:
       options:
         -
         - 3.0.0
+        - 3.1.0
         - Other (please provide detail below)
     validations:
       required: true

.github/pull_request_template.md

@@ -0,0 +1,22 @@
+## Description
+
+<!-- 
+Explain the purpose of the pull request. Be brief or detailed depending on the scope of the changes.
+-->
+
+## Related Issues
+
+<!-- 
+Optionally, list any related issues that this pull request addresses.
+-->
+
+## Checklist
+
+- [ ] I have read and followed the [CONTRIBUTING.md](https://github.com/Security-Onion-Solutions/securityonion/blob/3/main/CONTRIBUTING.md) file.
+- [ ] I have read and agree to the terms of the [Contributor License Agreement](https://securityonionsolutions.com/cla)
+
+## Questions or Comments
+
+<!--
+If you have any questions or comments about this pull request, add them here.
+-->

.github/workflows/contrib.yml

@@ -1,24 +0,0 @@
-name: contrib
-on:
-  issue_comment:
-    types: [created]
-  pull_request_target:
-    types: [opened,closed,synchronize]
-
-jobs:
-  CLAssistant:
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Contributor Check"
-        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: cla-assistant/github-action@v2.3.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-        with:
-          path-to-signatures: 'signatures_v1.json'
-          path-to-document: 'https://securityonionsolutions.com/cla' 
-          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
-          remote-organization-name: Security-Onion-Solutions
-          remote-repository-name: licensing
-

CONTRIBUTING.md

@@ -23,7 +23,7 @@
 
 * Link the PR to the related issue, either using [keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in the PR description, or [manually](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue).
 
-* **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution.
+* **Pull requests should be opened against the current `?/dev` branch of this repo**, and should clearly describe the problem and solution.
 
 * Be sure you have tested your changes and are confident they will not break other parts of the product.
 

VERSION

@@ -1 +1 @@
-3.0.0
+3.1.0

salt/global/soc_global.yaml

@@ -11,18 +11,14 @@ global:
     regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
     description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
-    regex: ^(ZEEK|SURICATA)$
     options:
       - ZEEK
       - SURICATA
-    regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
   pcapengine:
     description: Which engine to use for generating pcap. Currently only SURICATA is supported.
-    regex: ^(SURICATA)$
     options:
       - SURICATA
-    regexFailureMessage: You must enter either SURICATA.
     global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.
@@ -42,11 +38,9 @@ global:
     advanced: True
   pipeline:
     description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license.
-    regex: ^(REDIS|KAFKA)$
     options:
       - REDIS
       - KAFKA
-    regexFailureMessage: You must enter either REDIS or KAFKA.
     global: True
     advanced: True
   repo_host:

salt/influxdb/soc_influxdb.yaml

@@ -85,7 +85,10 @@ influxdb:
       description: The log level to use for outputting log statements. Allowed values are debug, info, or error.
       global: True
       advanced: false
-      regex: ^(info|debug|error)$
+      options:
+      - info
+      - debug
+      - error
       helpLink: influxdb
     metrics-disabled:
       description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible.
@@ -140,7 +143,9 @@ influxdb:
       description: Determines the type of storage used for secrets. Allowed values are bolt or vault.
       global: True
       advanced: True
-      regex: ^(bolt|vault)$
+      options:
+      - bolt
+      - vault
       helpLink: influxdb
     session-length:
       description: Number of minutes that a user login session can remain authenticated. 
@@ -260,7 +265,9 @@ influxdb:
       description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations.
       global: True
       advanced: True
-      regex: ^(disk|memory)$
+      options:
+      - disk
+      - memory
       helpLink: influxdb
     tls-cert: 
       description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses.

salt/kafka/soc_kafka.yaml

@@ -131,7 +131,10 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        regex: ^(JKS|PKCS12|PEM)$
+        options:
+        - JKS
+        - PKCS12
+        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.
@@ -160,7 +163,11 @@ kafka:
       security_x_protocol:
         description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT'
         title: security.protocol
-        regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT)
+        options:
+        - SASL_SSL
+        - PLAINTEXT
+        - SSL
+        - SASL_PLAINTEXT
         helpLink: kafka
       ssl_x_keystore_x_location:
         description: The key store file location within the Docker container.
@@ -174,7 +181,10 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        regex: ^(JKS|PKCS12|PEM)$
+        options:
+        - JKS
+        - PKCS12
+        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.

salt/kratos/soc_kratos.yaml

@@ -21,8 +21,12 @@ kratos:
         description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft"
         global: True
         forcedType: string
-        regex: "auth0|generic|github|google|microsoft"
+        options:
-        regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft"
+        - auth0
+        - generic
+        - github
+        - google
+        - microsoft
         helpLink: oidc
       client_id: 
         description: Specify the client ID, also referenced as the application ID. Required.
@@ -43,8 +47,9 @@ kratos:
         description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'.
         global: True
         forcedType: string
-        regex: me|userinfo
+        options:
-        regexFailureMessage: "Valid values are: me, userinfo"
+        - me
+        - userinfo
         helpLink: oidc
       auth_url: 
         description: Provider's auth URL. Required when provider is 'generic'.

salt/manager/tools/sbin/soup

@@ -305,7 +305,7 @@ clone_to_tmp() {
   # Make a temp location for the files
   mkdir -p /tmp/sogh
   cd /tmp/sogh
-  SOUP_BRANCH="-b 2.4/main"
+  SOUP_BRANCH="-b 3/main"
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi

salt/soc/defaults.yaml

@@ -2687,4 +2687,5 @@ soc:
               lowBalanceColorAlert: 500000
               enabled: true
               adapter: SOAI
+              charsPerTokenEstimate: 4
             

salt/soc/soc_soc.yaml

@@ -761,7 +761,7 @@ soc:
               required: True
             - field: origin
               label: Country of Origin for the Model Training
-              required: false
+              required: False
             - field: contextLimitSmall
               label: Context Limit (Small)
               forcedType: int
@@ -779,6 +779,10 @@ soc:
             - field: enabled
               label: Enabled
               forcedType: bool
+            - field: charsPerTokenEstimate
+              label: Characters per Token Estimate
+              forcedType: float
+              required: False
         apiTimeoutMs:
           description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
           global: True

salt/suricata/map.jinja

@@ -33,7 +33,7 @@
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %}
 {#   multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #}
-{%   set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %}
+{%   set maxfiles = ([1, (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round(0, 'ceil') | int] | max) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %}
 {% endif %}
 

salt/suricata/soc_suricata.yaml

@@ -64,8 +64,10 @@ suricata:
       helpLink: suricata
     conditional: 
       description: Set to "all" to record PCAP for all flows. Set to "alerts" to only record PCAP for Suricata alerts. Set to "tag" to only record PCAP for tagged rules.
-      regex: ^(all|alerts|tag)$
+      options:
-      regexFailureMessage: You must enter either all, alert or tag.
+      - all
+      - alerts
+      - tag
       helpLink: suricata
     dir:
       description: Parent directory to store PCAP.
@@ -83,7 +85,9 @@ suricata:
         advanced: True
       cluster-type:
         advanced: True
-        regex: ^(cluster_flow|cluster_qm)$
+        options:
+        - cluster_flow
+        - cluster_qm
       defrag:
         description: Enable defragmentation of IP packets before processing.
         forcedType: bool

salt/zeek/soc_zeek.yaml

@@ -5,7 +5,7 @@ zeek:
     helpLink: zeek
   ja4plus:
     enabled:
-      description: "Enables JA4+ fingerprinting (JA4S, JA4D, JA4H, JA4L, JA4SSH, JA4T, JA4TS, JA4X). By enabling this, you agree to the terms of the JA4+ license  [https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4](https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4)."
+      description: "Enables JA4+ fingerprinting (JA4S, JA4D, JA4H, JA4L, JA4SSH, JA4T, JA4TS, JA4X). By enabling this, you agree to the terms of the JA4+ license  [https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE](https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE)."
       forcedType: bool
       helpLink: zeek
       advanced: False