Merge pull request #7465 from Security-Onion-Solutions/fix/kibana_saved_objects_load

Kibana dashboard/saved objects loading improvements
2025-12-07 17:52:46 +01:00 · 2022-03-08 12:22:55 -05:00
parent f64da9632f d392cb258c
commit d9c3160fbf
5 changed files with 93 additions and 40 deletions
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -12,48 +12,73 @@ fi

 }

+RETURN_CODE=0
+
 import() {
-  local file=$1
-  ndjson_file=$(echo $file | sed -e "s/\.template$//")
-  # Copy template file
-  if [ "$file" != "$ndjson_file" ]; then
-    cp "$file" "$ndjson_file"
-  fi
+  local BASENAME=$(basename $1 | cut -d'.' -f1)
+  if [ ! -f "/opt/so/state/kibana_$BASENAME.txt" ]; then
+    local file=$1
+    ndjson_file=$(echo $file | sed -e "s/\.template$//")
+    # Copy template file
+    if [ "$file" != "$ndjson_file" ]; then
+      cp "$file" "$ndjson_file"
+    fi

-  # SOCtopus and Manager
-  if grep -lq 'PLACEHOLDER' "$ndjson_file"; then
-    sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" "$ndjson_file"
-  fi
-  
-  # Endgame
-  if grep -lq 'ENDGAMEHOST' "$ndjson_file"; then
-    sed -i "s/ENDGAMEHOST/{{ ENDGAMEHOST }}/g" "$ndjson_file"
-  fi
- 
-  wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
+    # SOCtopus and Manager
+    if grep -lq 'PLACEHOLDER' "$ndjson_file"; then
+      sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" "$ndjson_file"
+    fi

-  SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-  # Load saved objects
-  {{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@"$ndjson_file" >> /opt/so/log/kibana/misc.log
+    # Endgame
+    if grep -lq 'ENDGAMEHOST' "$ndjson_file"; then
+      sed -i "s/ENDGAMEHOST/{{ ENDGAMEHOST }}/g" "$ndjson_file"
+    fi
+
+    wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
+    RETURN_CODE=$?
+
+    SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+    # Load saved objects
+    RESPONSE=$({{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@"$ndjson_file")
+    echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
+
+    if [[ "$RETURN_CODE" != "1" ]]; then
+      touch /opt/so/state/kibana_$BASENAME.txt
+    fi
+  else
+    exit $RETURN_CODE
+  fi
 }

 update() {
-  wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"  
-  IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
-  for i in "${LINES[@]}"; do
-    {{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i "
-  done
+  local BASENAME=$(basename $1 | cut -d'.' -f1)
+  if [ ! -f "/opt/so/state/kibana_$BASENAME.txt" ]; then
+    wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
+    RETURN_CODE=$?

+    IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
+    for i in "${LINES[@]}"; do
+      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
+    done
+ 
+    if [[ "$RETURN_CODE" != "1" ]]; then
+      touch /opt/so/state/kibana_$BASENAME.txt
+    fi
+  else
+    exit $RETURN_CODE
+  fi
 }

 usage()
 {
 cat <<EOF
-Security Onion Kibana Config Loader
-  Options:
-  -h                   This message
-  -i <filename>        Import saved objects
-  -u <filename>        Update saved objects
+  Security Onion Kibana Config Loader
+    Options:
+    -h                   This message
+    -i <filename>        Import saved objects
+    -u <filename>        Update saved objects
 EOF
 }

--- a/salt/kibana/so_config_load.sls
+++ b/salt/kibana/so_config_load.sls
@@ -3,11 +3,18 @@ include:

 config_saved_objects:
  file.managed:
-    - name: /opt/so/conf/kibana/config_saved_objects.ndjson
+    - name: /opt/so/conf/kibana/config_saved_objects.ndjson.template
    - source: salt://kibana/files/config_saved_objects.ndjson
    - user: 932
    - group: 939

+config_saved_objects_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_config_saved_objects.txt
+    - onchanges:
+      - file: config_saved_objects
+
 so-kibana-config-load:
  cmd.run:
    - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/config_saved_objects.ndjson
--- a/salt/kibana/so_dashboard_load.sls
+++ b/salt/kibana/so_dashboard_load.sls
@@ -4,12 +4,19 @@ include:

 dashboard_saved_objects_template:
  file.managed:
-    - name: /opt/so/conf/kibana/saved_objects.ndjson
+    - name: /opt/so/conf/kibana/saved_objects.ndjson.template
    - source: salt://kibana/files/saved_objects.ndjson
    - user: 932
    - group: 939
    - show_changes: False

+dashboard_saved_objects_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_saved_objects.txt
+    - onchanges:
+      - file: dashboard_saved_objects_template
+
 so-kibana-dashboard-load:
  cmd.run:
    - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/saved_objects.ndjson
@@ -20,12 +27,19 @@ so-kibana-dashboard-load:
 {%- if HIGHLANDER %}
 dashboard_saved_objects_template_hl:
  file.managed:
-    - name: /opt/so/conf/kibana/hl.ndjson
+    - name: /opt/so/conf/kibana/hl.ndjson.template
    - source: salt://kibana/files/hl.ndjson
    - user: 932
    - group: 939
    - show_changes: False

+dashboard_saved_objects_hl_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_hl.txt
+    - onchanges:
+      - file: dashboard_saved_objects_template_hl
+
 so-kibana-dashboard-load_hl:
  cmd.run:
    - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/hl.ndjson
--- a/salt/kibana/so_securitySolution_load.sls
+++ b/salt/kibana/so_securitySolution_load.sls
@@ -3,11 +3,18 @@ include:

 securitySolution_saved_objects:
  file.managed:
-    - name: /opt/so/conf/kibana/securitySolution_saved_objects.ndjson
+    - name: /opt/so/conf/kibana/securitySolution_saved_objects.ndjson.template
    - source: salt://kibana/files/securitySolution_saved_objects.ndjson
    - user: 932
    - group: 939

+securitySolution_saved_objects_changes:
+  file.absent:
+    - names:
+      - /opt/so/state/kibana_config_saved_objects.txt
+    - onchanges:
+      - file: securitySolution_saved_objects
+
 so-kibana-securitySolution_saved_objects-load:
  cmd.run:
    - name: /usr/sbin/so-kibana-config-load -u /opt/so/conf/kibana/securitySolution_saved_objects.ndjson
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -118,7 +118,7 @@ base:
    - elasticsearch
    {%- endif %}
    {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
    {%- endif %}
    - pcap
    - suricata
@@ -193,7 +193,7 @@ base:
    - redis
    {%- endif %}
    {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
    {%- endif %}
    - curator
    {%- if ELASTALERT %}
@@ -260,7 +260,7 @@ base:
    - redis
    {%- endif %}
    {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
    {%- endif %}
    - pcap
    - suricata
@@ -361,7 +361,7 @@ base:
    {%- endif %}
    - curator
    {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
    {%- endif %}
    {%- if ELASTALERT %}
    - elastalert
@@ -467,7 +467,7 @@ base:
    - elasticsearch
    {%- endif %}
    {%- if KIBANA %}
-    - kibana
+    - kibana.so_savedobjects_defaults
    {%- endif %}
    {%- if FILEBEAT %}
    - filebeat
@@ -519,4 +519,4 @@ base:
    - schedule
    - docker_clean
    - filebeat
-    - idh
+    - idh