Organized, Annotated

Settling on the final shape of the new config values.

salt/common/tools/sbin/so-common

@@ -404,6 +404,25 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }
 
+initialize_elasticsearch_indices() {
+    local index_names=$1
+    local default_entry=${2:-'{"@timestamp":"0"}'}
+
+    for idx in $index_names; do
+        if ! so-elasticsearch-query "$idx" --fail --retry 3 --retry-delay 30 >/dev/null 2>&1; then
+            echo "Index does not already exist. Initializing $idx index."
+
+            if retry 3 10 "so-elasticsearch-query "$idx/_doc" -d '$default_entry' -XPOST --fail 2>/dev/null" '"successful":1'; then
+                echo "Successfully initialized $idx index."
+            else
+                echo "Failed to initialize $idx index after 3 attempts."
+            fi
+        else
+            echo "Index $idx already exists. No action needed."
+        fi
+    done
+}
+
 lookup_bond_interfaces() {
 	cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
 }

salt/elasticfleet/config.sls

@@ -11,6 +11,7 @@
 
 include:
   - elasticfleet.artifact_registry
+  - elasticfleet.ssl
 
 # Add EA Group
 elasticfleetgroup:

salt/elasticfleet/enabled.sls

@@ -15,7 +15,6 @@
 include:
   - ca
   - logstash.ssl
-  - elasticfleet.ssl
   - elasticfleet.config
   - elasticfleet.sostatus
 

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -22,7 +22,7 @@
             "data_stream.dataset": "import",
             "pipeline": "",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
-            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}).log$"],
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
             "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
             "tags": [],

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -21,7 +21,7 @@
             ],
             "data_stream.dataset": "zeek",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
-            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}).log$"],
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
             "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
             "tags": [],

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -14,7 +14,7 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%20:%20%22{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%22%20and%20policy_id%20:%20%22so-grid-nodes_general%22&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -34,6 +34,11 @@ if [[ "$RETURN_CODE" != "0" ]]; then
   exit 1
 fi
 
+if [[ ! -f /etc/pki/elasticfleet-kafka.crt || ! -f /etc/pki/elasticfleet-kafka.key ]]; then
+  echo -e "\nKafka certificates not found, can't setup Elastic Fleet output policy for Kafka...\n"
+  exit 1
+fi
+
 KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
 KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
 KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)

salt/manager/tools/sbin/soup

@@ -692,6 +692,8 @@ post_to_2.4.210() {
 
   disable_redis_heavynodes
 
+  initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat"
+
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 

salt/soc/defaults.yaml

@@ -1494,10 +1494,16 @@ soc:
                 branch: main
                 folder: securityonion-normalized
         assistant:
-          apiUrl: https://onionai.securityonion.net
-          healthTimeoutSeconds: 3
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000
+          adapters:
+            securityonion_ai_cloud:
+              apiUrl: https://onionai.securityonion.net
+              healthTimeoutSeconds: 5
+            gemini:
+              apiKey: ""
+              serviceAccountJSON: ""
+              serviceAccountLocation: ""
         salt:
           queueDir: /opt/sensoroni/queue
           timeoutMs: 45000
@@ -2659,6 +2665,7 @@ soc:
               contextLimitLarge: 1000000
               lowBalanceColorAlert: 500000
               enabled: true
+              adapter: bedrock
             - id: qwen-235b
               displayName: QWEN 235B ($)
               origin: China
@@ -2666,3 +2673,4 @@ soc:
               contextLimitLarge: 256000
               lowBalanceColorAlert: 500000
               enabled: true
+              adapter: bedrock

salt/soc/soc_soc.yaml

@@ -650,13 +650,6 @@ soc:
                 label: Folder
             airgap: *pbRepos
         assistant:
-          apiUrl:
-            description: The URL of the AI gateway.
-            global: True
-          healthTimeoutSeconds:
-            description: Timeout in seconds for the Onion AI health check.
-            global: True
-            advanced: True
           systemPromptAddendum:
             description: Additional context to provide to the AI assistant about this SOC deployment. This can include information about your environment, policies, or any other relevant details that can help the AI provide more accurate and tailored assistance. Long prompts may be shortened.
             global: True
@@ -666,6 +659,26 @@ soc:
             description: Maximum length of the system prompt addendum. Longer prompts will be truncated.
             global: True
             advanced: True
+          adapters:
+            securityonion_ai_cloud:
+              apiUrl:
+                description: The URL of the AI gateway.
+                global: True
+              healthTimeoutSeconds:
+                description: Timeout in seconds for the Onion AI health check.
+                global: True
+                advanced: True
+            gemini:
+              apiKey:
+                description: A Google API key generated in Google AI Studio or the Google Cloud Console.
+                global: True
+              serviceAccountJSON:
+                description: The JSON file contents holding the credentials for a Google Service Account. Uses Vertex AI.
+                global: True
+                multiline: True
+              serviceAccountLocation:
+                description: The location of your project in Google Cloud. Example: us-central1
+                global: True
       client:
         assistant:
           enabled:

setup/so-setup

@@ -828,6 +828,7 @@ if ! [[ -f $install_opt_file ]]; then
 		fi
 		checkin_at_boot
 		set_initial_firewall_access
+        initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat"
 		# run a final highstate before enabling scheduled highstates. 
 		# this will ensure so-elasticsearch-ilm-policy-load and so-elasticsearch-templates-load have a chance to run after elasticfleet is setup
 		info "Running final highstate for setup"