Merge branch 'dev' into foxtrot

salt/common/tools/sbin/soup

@@ -735,6 +735,22 @@ if [[ -n $lsl_msg ]]; then
   esac
 fi
 
+NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+
+if [ $NUM_MINIONS -gt 1 ]; then
+
+	cat << EOF
+This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
+
+Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
+
+If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
+
+For more information, please see https://docs.securityonion.net/en/2.3/soup.html#distributed-deployments.
+EOF
+
+fi
+
 }
 
 main "$@" | tee /dev/fd/3

salt/kibana/bin/so-kibana-config-load

@@ -3,6 +3,8 @@
 # {%- set FLEET_NODE = salt['pillar.get']('global:fleet_node', False) -%}
 # {%- set MANAGER = salt['pillar.get']('global:url_base', '') %}
 
+. /usr/sbin/so-common
+
 # Copy template file
 cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_objects.ndjson
 
@@ -14,9 +16,11 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o
 # SOCtopus and Manager
 sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson
 
+wait_for_web_response "http://localhost:5601/app/kibana" "Elastic"
 ## This hackery will be removed if using Elastic Auth ##
 
 # Let's snag a cookie from Kibana
 THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
 # Load saved objects
 curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1

salt/kibana/init.sls

@@ -94,21 +94,10 @@ kibanadashtemplate:
     - user: 932
     - group: 939
 
-wait_for_kibana:
-  module.run:
-    - http.wait_for_successful_query:
-      - url: "http://{{MANAGER}}:5601/api/saved_objects/_find?type=config"
-      - wait_for: 900
-    - onchanges:
-      - file: kibanadashtemplate
-
 so-kibana-config-load:
   cmd.run:
     - name: /usr/sbin/so-kibana-config-load
     - cwd: /opt/so
-    - onchanges:
-      - wait_for_kibana
-
 
 # Keep the setting correct
 #KibanaHappy:

salt/soc/files/soc/soc.json

@@ -2,6 +2,10 @@
 {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %}
 {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
 {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
+{%- set API_TIMEOUT = salt['pillar.get']('sensoroni:api_timeout_ms', 0) %}
+{%- set WEBSOCKET_TIMEOUT = salt['pillar.get']('sensoroni:websocket_timeout_ms', 0) %}
+{%- set TIP_TIMEOUT = salt['pillar.get']('sensoroni:tip_timeout_ms', 0) %}
+{%- set CACHE_EXPIRATION = salt['pillar.get']('sensoroni:cache_expiration_ms', 0) %}
 {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %}
 {%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %}
 {%- import_json "soc/files/soc/alerts.eventfields.json" as alerts_eventfields %}
@@ -66,6 +70,10 @@
       "docsUrl": "https://docs.securityonion.net/en/2.3/",
       "cheatsheetUrl": "https://github.com/Security-Onion-Solutions/securityonion-docs/raw/2.3/images/cheat-sheet/Security-Onion-Cheat-Sheet.pdf",
       {%- endif %}
+      "apiTimeoutMs": {{ API_TIMEOUT }},
+      "webSocketTimeoutMs": {{ WEBSOCKET_TIMEOUT }},
+      "tipTimeoutMs": {{ TIP_TIMEOUT }},
+      "cacheExpirationMs": {{ CACHE_EXPIRATION }},
       "hunt": {
         "advanced": true,
         "groupItemsPerPage": 10,