Merge pull request #7465 from Security-Onion-Solutions/fix/kibana_saved_objects_load

Kibana dashboard/saved objects loading improvements

salt/kibana/bin/so-kibana-config-load

@@ -12,48 +12,73 @@ fi
 
 }
 
+RETURN_CODE=0
+
 import() {
-  local file=$1
+  local BASENAME=$(basename $1 | cut -d'.' -f1)
-  ndjson_file=$(echo $file | sed -e "s/\.template$//")
+  if [ ! -f "/opt/so/state/kibana_$BASENAME.txt" ]; then
-  # Copy template file
+    local file=$1
-  if [ "$file" != "$ndjson_file" ]; then
+    ndjson_file=$(echo $file | sed -e "s/\.template$//")
-    cp "$file" "$ndjson_file"
+    # Copy template file
+    if [ "$file" != "$ndjson_file" ]; then
+      cp "$file" "$ndjson_file"
+    fi
+
+    # SOCtopus and Manager
+    if grep -lq 'PLACEHOLDER' "$ndjson_file"; then
+      sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" "$ndjson_file"
+    fi
+
+    # Endgame
+    if grep -lq 'ENDGAMEHOST' "$ndjson_file"; then
+      sed -i "s/ENDGAMEHOST/{{ ENDGAMEHOST }}/g" "$ndjson_file"
+    fi
+
+    wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
+    RETURN_CODE=$?
+
+    SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+    # Load saved objects
+    RESPONSE=$({{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@"$ndjson_file")
+    echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
+
+    if [[ "$RETURN_CODE" != "1" ]]; then
+      touch /opt/so/state/kibana_$BASENAME.txt
+    fi
+  else
+    exit $RETURN_CODE
   fi
-
-  # SOCtopus and Manager
-  if grep -lq 'PLACEHOLDER' "$ndjson_file"; then
-    sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" "$ndjson_file"
-  fi
-  
-  # Endgame
-  if grep -lq 'ENDGAMEHOST' "$ndjson_file"; then
-    sed -i "s/ENDGAMEHOST/{{ ENDGAMEHOST }}/g" "$ndjson_file"
-  fi
- 
-  wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
-
-  SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-  # Load saved objects
-  {{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@"$ndjson_file" >> /opt/so/log/kibana/misc.log
 }
 
 update() {
-  wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"  
+  local BASENAME=$(basename $1 | cut -d'.' -f1)
-  IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
+  if [ ! -f "/opt/so/state/kibana_$BASENAME.txt" ]; then
-  for i in "${LINES[@]}"; do
+    wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
-    {{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i "
+    RETURN_CODE=$?
-  done
 
+    IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
+    for i in "${LINES[@]}"; do
+      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
+    done
+ 
+    if [[ "$RETURN_CODE" != "1" ]]; then
+      touch /opt/so/state/kibana_$BASENAME.txt
+    fi
+  else
+    exit $RETURN_CODE
+  fi
 }
 
 usage()
 {
 cat <<EOF
-Security Onion Kibana Config Loader
+  Security Onion Kibana Config Loader
-  Options:
+    Options:
-  -h                   This message
+    -h                   This message
-  -i <filename>        Import saved objects
+    -i <filename>        Import saved objects
-  -u <filename>        Update saved objects
+    -u <filename>        Update saved objects
 EOF
 }
 

salt/kibana/so_config_load.sls

@@ -3,11 +3,18 @@ include:
 
 config_saved_objects:
   file.managed:
-    - name: /opt/so/conf/kibana/config_saved_objects.ndjson
+    - name: /opt/so/conf/kibana/config_saved_objects.ndjson.template
     - source: salt://kibana/files/config_saved_objects.ndjson
     - user: 932
     - group: 939
 
+config_saved_objects_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_config_saved_objects.txt
+    - onchanges:
+      - file: config_saved_objects
+
 so-kibana-config-load:
   cmd.run:
     - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/config_saved_objects.ndjson

salt/kibana/so_dashboard_load.sls

@@ -4,12 +4,19 @@ include:
 
 dashboard_saved_objects_template:
   file.managed:
-    - name: /opt/so/conf/kibana/saved_objects.ndjson
+    - name: /opt/so/conf/kibana/saved_objects.ndjson.template
     - source: salt://kibana/files/saved_objects.ndjson
     - user: 932
     - group: 939
     - show_changes: False
 
+dashboard_saved_objects_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_saved_objects.txt
+    - onchanges:
+      - file: dashboard_saved_objects_template
+
 so-kibana-dashboard-load:
   cmd.run:
     - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/saved_objects.ndjson
@@ -20,12 +27,19 @@ so-kibana-dashboard-load:
 {%- if HIGHLANDER %}
 dashboard_saved_objects_template_hl:
   file.managed:
-    - name: /opt/so/conf/kibana/hl.ndjson
+    - name: /opt/so/conf/kibana/hl.ndjson.template
     - source: salt://kibana/files/hl.ndjson
     - user: 932
     - group: 939
     - show_changes: False
 
+dashboard_saved_objects_hl_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_hl.txt
+    - onchanges:
+      - file: dashboard_saved_objects_template_hl
+
 so-kibana-dashboard-load_hl:
   cmd.run:
     - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/hl.ndjson

salt/kibana/so_securitySolution_load.sls

@@ -3,11 +3,18 @@ include:
 
 securitySolution_saved_objects:
   file.managed:
-    - name: /opt/so/conf/kibana/securitySolution_saved_objects.ndjson
+    - name: /opt/so/conf/kibana/securitySolution_saved_objects.ndjson.template
     - source: salt://kibana/files/securitySolution_saved_objects.ndjson
     - user: 932
     - group: 939
 
+securitySolution_saved_objects_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_config_saved_objects.txt
+    - onchanges:
+      - file: securitySolution_saved_objects
+
 so-kibana-securitySolution_saved_objects-load:
   cmd.run:
     - name: /usr/sbin/so-kibana-config-load -u /opt/so/conf/kibana/securitySolution_saved_objects.ndjson

salt/top.sls

@@ -118,7 +118,7 @@ base:
     - elasticsearch
     {%- endif %}
     {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
     {%- endif %}
     - pcap
     - suricata
@@ -193,7 +193,7 @@ base:
     - redis
     {%- endif %}
     {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
     {%- endif %}
     - curator
     {%- if ELASTALERT %}
@@ -260,7 +260,7 @@ base:
     - redis
     {%- endif %}
     {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
     {%- endif %}
     - pcap
     - suricata
@@ -361,7 +361,7 @@ base:
     {%- endif %}
     - curator
     {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
     {%- endif %}
     {%- if ELASTALERT %}
     - elastalert
@@ -467,7 +467,7 @@ base:
     - elasticsearch
     {%- endif %}
     {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
     {%- endif %}
     {%- if FILEBEAT %}
     - filebeat