From d10d4acf9f06184710549245cd6a64bea602abb1 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 14:33:15 +0000 Subject: [PATCH 1/8] Modify Kibana config load script to drop file if successfully executed --- salt/kibana/bin/so-kibana-config-load | 84 +++++++++++++++++---------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index d98b0e85f..51a30c911 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -12,48 +12,72 @@ fi } +RETURN_CODE=0 + import() { - local file=$1 - ndjson_file=$(echo $file | sed -e "s/\.template$//") - # Copy template file - if [ "$file" != "$ndjson_file" ]; then - cp "$file" "$ndjson_file" - fi + local BASENAME=$(basename $1 | cut -d'.' -f1) + if [ ! -f "/opt/so/state/kibana_$BASENAME.txt" ]; then + local file=$1 + ndjson_file=$(echo $file | sed -e "s/\.template$//") + # Copy template file + if [ "$file" != "$ndjson_file" ]; then + cp "$file" "$ndjson_file" + fi - # SOCtopus and Manager - if grep -lq 'PLACEHOLDER' "$ndjson_file"; then - sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" "$ndjson_file" - fi - - # Endgame - if grep -lq 'ENDGAMEHOST' "$ndjson_file"; then - sed -i "s/ENDGAMEHOST/{{ ENDGAMEHOST }}/g" "$ndjson_file" - fi - - wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" + # SOCtopus and Manager + if grep -lq 'PLACEHOLDER' "$ndjson_file"; then + sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" "$ndjson_file" + fi - SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - # Load saved objects - {{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@"$ndjson_file" >> /opt/so/log/kibana/misc.log + # Endgame + if grep -lq 'ENDGAMEHOST' "$ndjson_file"; then + sed -i "s/ENDGAMEHOST/{{ ENDGAMEHOST }}/g" "$ndjson_file" + fi + + wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" + RETURN_CODE=$? + + SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + + # Load saved objects + RESPONSE=$({{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@"$ndjson_file") + echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1;fi + + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/kibana_$BASENAME.txt + fi + else + exit $RETURN_CODE + fi } update() { - wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" - IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' - for i in "${LINES[@]}"; do - {{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i " - done + local BASENAME=$(basename $1 | cut -d'.' -f1) + if [ ! -f "/opt/so/state/kibana_$BASENAME.txt" ]; then + wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" + RETURN_CODE=$? + IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' + for i in "${LINES[@]}"; do + RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi + done + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/kibana_$BASENAME.txt + fi + else + exit $RETURN_CODE + fi } usage() { cat < Import saved objects - -u Update saved objects + Security Onion Kibana Config Loader + Options: + -h This message + -i Import saved objects + -u Update saved objects EOF } From c0f49f6fb0e7e8013d4eadf9be14f9fdbca4ac20 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 14:35:04 +0000 Subject: [PATCH 2/8] Remove drop file when dashbaord saved objects change --- salt/kibana/so_dashboard_load.sls | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/kibana/so_dashboard_load.sls b/salt/kibana/so_dashboard_load.sls index a340682bf..965c245b6 100644 --- a/salt/kibana/so_dashboard_load.sls +++ b/salt/kibana/so_dashboard_load.sls @@ -10,6 +10,13 @@ dashboard_saved_objects_template: - group: 939 - show_changes: False +dashboard_saved_objects_changes: + file.absent: + - names: + - /opt/so/state/kibana_saved_objects.txt + - onchanges: + - file: dashboard_saved_objects_template + so-kibana-dashboard-load: cmd.run: - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/saved_objects.ndjson @@ -26,6 +33,13 @@ dashboard_saved_objects_template_hl: - group: 939 - show_changes: False +dashboard_saved_objects_hl_changes: + file.absent: + - names: + - /opt/so/state/kibana_hl.txt + - onchanges: + - file: dashboard_saved_objects_template_hl + so-kibana-dashboard-load_hl: cmd.run: - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/hl.ndjson From 14dddd86490ed11fc0abb1b786b129ee896ef815 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 14:37:15 +0000 Subject: [PATCH 3/8] Remove drop file when config saved objects change --- salt/kibana/so_config_load.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/kibana/so_config_load.sls b/salt/kibana/so_config_load.sls index 58cd0ffbb..59f07e1da 100644 --- a/salt/kibana/so_config_load.sls +++ b/salt/kibana/so_config_load.sls @@ -8,6 +8,13 @@ config_saved_objects: - user: 932 - group: 939 +config_saved_objects_changes: + file.absent: + - names: + - /opt/so/state/kibana_config_saved_objects.txt + - onchanges: + - file: config_saved_objects + so-kibana-config-load: cmd.run: - name: /usr/sbin/so-kibana-config-load -i /opt/so/conf/kibana/config_saved_objects.ndjson From 28554164cd910d151176c3e28a29791c9d8b1091 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 14:39:23 +0000 Subject: [PATCH 4/8] Remove drop file when securitySolution saved objects change --- salt/kibana/so_securitySolution_load.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/kibana/so_securitySolution_load.sls b/salt/kibana/so_securitySolution_load.sls index 49dbfa25b..28eb0bc2a 100644 --- a/salt/kibana/so_securitySolution_load.sls +++ b/salt/kibana/so_securitySolution_load.sls @@ -8,6 +8,13 @@ securitySolution_saved_objects: - user: 932 - group: 939 +securitySolution_saved_objects_changes: + file.absent: + - names: + - /opt/so/state/kibana_config_saved_objects.txt + - onchanges: + - file: securitySolution_saved_objects + so-kibana-securitySolution_saved_objects-load: cmd.run: - name: /usr/sbin/so-kibana-config-load -u /opt/so/conf/kibana/securitySolution_saved_objects.ndjson From 3f31f7fd4138c36b17cd182feab200b2b0f50c04 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 16:43:43 +0000 Subject: [PATCH 5/8] Add .template extension to fix script behavior and not modify watched file --- salt/kibana/so_dashboard_load.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/kibana/so_dashboard_load.sls b/salt/kibana/so_dashboard_load.sls index 965c245b6..bdc3dd74e 100644 --- a/salt/kibana/so_dashboard_load.sls +++ b/salt/kibana/so_dashboard_load.sls @@ -4,7 +4,7 @@ include: dashboard_saved_objects_template: file.managed: - - name: /opt/so/conf/kibana/saved_objects.ndjson + - name: /opt/so/conf/kibana/saved_objects.ndjson.template - source: salt://kibana/files/saved_objects.ndjson - user: 932 - group: 939 @@ -27,7 +27,7 @@ so-kibana-dashboard-load: {%- if HIGHLANDER %} dashboard_saved_objects_template_hl: file.managed: - - name: /opt/so/conf/kibana/hl.ndjson + - name: /opt/so/conf/kibana/hl.ndjson.template - source: salt://kibana/files/hl.ndjson - user: 932 - group: 939 From a6fd1023b45c8fc036ac33ecb868b1abce5956ab Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 16:57:26 +0000 Subject: [PATCH 6/8] Fix criteria for successful execution --- salt/kibana/bin/so-kibana-config-load | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 51a30c911..4752925b4 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -41,7 +41,7 @@ import() { # Load saved objects RESPONSE=$({{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@"$ndjson_file") - echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1;fi + echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/kibana_$BASENAME.txt @@ -60,8 +60,9 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") - echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi + echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done + if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/kibana_$BASENAME.txt fi From 86e228b20045f1eb119aa76e128f7da11a8a0191 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 16:58:37 +0000 Subject: [PATCH 7/8] Add .template extension for future-proofing config files --- salt/kibana/so_config_load.sls | 2 +- salt/kibana/so_securitySolution_load.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/kibana/so_config_load.sls b/salt/kibana/so_config_load.sls index 59f07e1da..10587c68e 100644 --- a/salt/kibana/so_config_load.sls +++ b/salt/kibana/so_config_load.sls @@ -3,7 +3,7 @@ include: config_saved_objects: file.managed: - - name: /opt/so/conf/kibana/config_saved_objects.ndjson + - name: /opt/so/conf/kibana/config_saved_objects.ndjson.template - source: salt://kibana/files/config_saved_objects.ndjson - user: 932 - group: 939 diff --git a/salt/kibana/so_securitySolution_load.sls b/salt/kibana/so_securitySolution_load.sls index 28eb0bc2a..c425e975a 100644 --- a/salt/kibana/so_securitySolution_load.sls +++ b/salt/kibana/so_securitySolution_load.sls @@ -3,7 +3,7 @@ include: securitySolution_saved_objects: file.managed: - - name: /opt/so/conf/kibana/securitySolution_saved_objects.ndjson + - name: /opt/so/conf/kibana/securitySolution_saved_objects.ndjson.template - source: salt://kibana/files/securitySolution_saved_objects.ndjson - user: 932 - group: 939 From d392cb258cd32dd754f92083b32c17483a7cd7b4 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 8 Mar 2022 16:59:48 +0000 Subject: [PATCH 8/8] Switch Kibana state to kibana.so_savedobjects_defaults in top file --- salt/top.sls | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index fc8434e69..67d717d3b 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -118,7 +118,7 @@ base: - elasticsearch {%- endif %} {%- if KIBANA %} - - kibana + - kibana.so_savedobjects_defaults {%- endif %} - pcap - suricata @@ -193,7 +193,7 @@ base: - redis {%- endif %} {%- if KIBANA %} - - kibana + - kibana.so_savedobjects_defaults {%- endif %} - curator {%- if ELASTALERT %} @@ -260,7 +260,7 @@ base: - redis {%- endif %} {%- if KIBANA %} - - kibana + - kibana.so_savedobjects_defaults {%- endif %} - pcap - suricata @@ -361,7 +361,7 @@ base: {%- endif %} - curator {%- if KIBANA %} - - kibana + - kibana.so_savedobjects_defaults {%- endif %} {%- if ELASTALERT %} - elastalert @@ -467,7 +467,7 @@ base: - elasticsearch {%- endif %} {%- if KIBANA %} - - kibana + - kibana.so_savedobjects_defaults {%- endif %} {%- if FILEBEAT %} - filebeat @@ -519,4 +519,4 @@ base: - schedule - docker_clean - filebeat - - idh \ No newline at end of file + - idh