Merge pull request #15742 from Security-Onion-Solutions/mwright/ai-query-length

Assistant: charsPerTokenEstimate

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -10,6 +10,7 @@ body:
       options:
         -
         - 3.0.0
+        - 3.1.0
         - Other (please provide detail below)
     validations:
       required: true

VERSION

@@ -1 +1 @@
-3.0.0
+3.1.0

salt/common/tools/sbin/so-nsm-clear

@@ -66,22 +66,11 @@ delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"
   [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }
-delete_import() {
-  IMPORT_DATA="/nsm/import/"
-  [ -d $IMPORT_DATA ] && rm -rf $IMPORT_DATA/*
-}
-delete_strelka() {
-  STRELKA_HISTORY_DATA="/nsm/strelka/history/"
-  STRELKA_PROCESSED_DATA="/nsm/strelka/processed/"
-  [ -d $STRELKA_HISTORY_DATA ] && rm -rf $STRELKA_HISTORY_DATA/*
-  [ -d $STRELKA_PROCESSED_DATA ] && rm -rf $STRELKA_PROCESSED_DATA/*
-}
-
 
 so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
 so-suricata-start
-delete_import
+
-delete_strelka
+

salt/global/soc_global.yaml

@@ -11,18 +11,14 @@ global:
     regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
     description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
-    regex: ^(ZEEK|SURICATA)$
     options:
       - ZEEK
       - SURICATA
-    regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
   pcapengine:
     description: Which engine to use for generating pcap. Currently only SURICATA is supported.
-    regex: ^(SURICATA)$
     options:
       - SURICATA
-    regexFailureMessage: You must enter either SURICATA.
     global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.
@@ -42,11 +38,9 @@ global:
     advanced: True
   pipeline:
     description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license.
-    regex: ^(REDIS|KAFKA)$
     options:
       - REDIS
       - KAFKA
-    regexFailureMessage: You must enter either REDIS or KAFKA.
     global: True
     advanced: True
   repo_host:

salt/influxdb/soc_influxdb.yaml

@@ -85,7 +85,10 @@ influxdb:
       description: The log level to use for outputting log statements. Allowed values are debug, info, or error.
       global: True
       advanced: false
-      regex: ^(info|debug|error)$
+      options:
+      - info
+      - debug
+      - error
       helpLink: influxdb
     metrics-disabled:
       description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible.
@@ -140,7 +143,9 @@ influxdb:
       description: Determines the type of storage used for secrets. Allowed values are bolt or vault.
       global: True
       advanced: True
-      regex: ^(bolt|vault)$
+      options:
+      - bolt
+      - vault
       helpLink: influxdb
     session-length:
       description: Number of minutes that a user login session can remain authenticated. 
@@ -260,7 +265,9 @@ influxdb:
       description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations.
       global: True
       advanced: True
-      regex: ^(disk|memory)$
+      options:
+      - disk
+      - memory
       helpLink: influxdb
     tls-cert: 
       description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses.

salt/kafka/soc_kafka.yaml

@@ -131,7 +131,10 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        regex: ^(JKS|PKCS12|PEM)$
+        options:
+        - JKS
+        - PKCS12
+        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.
@@ -160,7 +163,11 @@ kafka:
       security_x_protocol:
         description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT'
         title: security.protocol
-        regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT)
+        options:
+        - SASL_SSL
+        - PLAINTEXT
+        - SSL
+        - SASL_PLAINTEXT
         helpLink: kafka
       ssl_x_keystore_x_location:
         description: The key store file location within the Docker container.
@@ -174,7 +181,10 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        regex: ^(JKS|PKCS12|PEM)$
+        options:
+        - JKS
+        - PKCS12
+        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.

salt/kratos/soc_kratos.yaml

@@ -21,8 +21,12 @@ kratos:
         description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft"
         global: True
         forcedType: string
-        regex: "auth0|generic|github|google|microsoft"
+        options:
-        regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft"
+        - auth0
+        - generic
+        - github
+        - google
+        - microsoft
         helpLink: oidc
       client_id: 
         description: Specify the client ID, also referenced as the application ID. Required.
@@ -43,8 +47,9 @@ kratos:
         description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'.
         global: True
         forcedType: string
-        regex: me|userinfo
+        options:
-        regexFailureMessage: "Valid values are: me, userinfo"
+        - me
+        - userinfo
         helpLink: oidc
       auth_url: 
         description: Provider's auth URL. Required when provider is 'generic'.

salt/soc/defaults.yaml

@@ -2687,4 +2687,5 @@ soc:
               lowBalanceColorAlert: 500000
               enabled: true
               adapter: SOAI
+              charsPerTokenEstimate: 4
             

salt/soc/soc_soc.yaml

@@ -761,7 +761,7 @@ soc:
               required: True
             - field: origin
               label: Country of Origin for the Model Training
-              required: false
+              required: False
             - field: contextLimitSmall
               label: Context Limit (Small)
               forcedType: int
@@ -779,6 +779,10 @@ soc:
             - field: enabled
               label: Enabled
               forcedType: bool
+            - field: charsPerTokenEstimate
+              label: Characters per Token Estimate
+              forcedType: float
+              required: False
         apiTimeoutMs:
           description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
           global: True

salt/suricata/map.jinja

@@ -33,7 +33,7 @@
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %}
 {#   multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #}
-{%   set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %}
+{%   set maxfiles = ([1, (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round(0, 'ceil') | int] | max) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %}
 {% endif %}
 

salt/suricata/soc_suricata.yaml

@@ -64,8 +64,10 @@ suricata:
       helpLink: suricata
     conditional: 
       description: Set to "all" to record PCAP for all flows. Set to "alerts" to only record PCAP for Suricata alerts. Set to "tag" to only record PCAP for tagged rules.
-      regex: ^(all|alerts|tag)$
+      options:
-      regexFailureMessage: You must enter either all, alert or tag.
+      - all
+      - alerts
+      - tag
       helpLink: suricata
     dir:
       description: Parent directory to store PCAP.
@@ -83,7 +85,9 @@ suricata:
         advanced: True
       cluster-type:
         advanced: True
-        regex: ^(cluster_flow|cluster_qm)$
+        options:
+        - cluster_flow
+        - cluster_qm
       defrag:
         description: Enable defragmentation of IP packets before processing.
         forcedType: bool