From b4bc32d3cab429886809be6141c04df5b090fc88 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 8 Dec 2021 14:33:15 -0500 Subject: [PATCH] set logstash pillar and enable avanced ls menu for so-receiver --- setup/so-functions | 28 ++++++++++++++-------------- setup/so-setup | 26 +++++++++++++++++++++----- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 527d914e5..b47f585cd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1327,15 +1327,6 @@ elasticsearch_pillar() { " log_size_limit: $log_size_limit"\ " node_route_type: 'hot'"\ "" >> "$pillar_file" - - printf '%s\n'\ - "logstash_settings:"\ - " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ - " ls_input_threads: $LSINPUTTHREADS"\ - " lsheap: $NODE_LS_HEAP_SIZE"\ - " ls_pipeline_workers: $num_cpu_cores"\ - "" >> "$pillar_file" - } es_heapsize() { @@ -1554,6 +1545,20 @@ import_registry_docker() { fi } +logstash_pillar() { + + local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls + + # Create the logstash pillar + printf '%s\n'\ + "logstash_settings:"\ + " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ + " ls_input_threads: $LSINPUTTHREADS"\ + " lsheap: $NODE_LS_HEAP_SIZE"\ + " ls_pipeline_workers: $num_cpu_cores"\ + "" >> "$pillar_file" +} + # Set Logstash heap size based on total memory ls_heapsize() { @@ -2407,11 +2412,6 @@ securityonion_repo() { fi } -set_base_heapsizes() { - es_heapsize - ls_heapsize -} - set_network_dev_status_list() { readarray -t nmcli_dev_status_list <<< "$(nmcli -t -f DEVICE,STATE -c no dev status)" export nmcli_dev_status_list diff --git a/setup/so-setup b/setup/so-setup index 5c1194df7..e890853f5 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -464,8 +464,14 @@ if [[ $is_helix || $is_manager || $is_import ]]; then collect_homenet_mngr fi +#set base elasticsearch heap size if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then - set_base_heapsizes + es_heapsize +fi + +#set base logstash heap size +if [[ $is_helix || $is_manager || $is_node || $is_import || $is_receiver ]]; then + ls_heapsize fi if [[ $is_manager && ! $is_eval ]]; then @@ -544,17 +550,21 @@ fi [[ $is_iso ]] && collect_ntp_servers -if [[ $is_node && ! $is_eval ]]; then +if [[ ($is_node || $is_receiver) && ! $is_eval ]]; then whiptail_node_advanced if [ "$NODESETUP" == 'NODEADVANCED' ]; then - collect_node_es_heap + if [[ ! $is_receiver ]]; then + collect_node_es_heap + collect_es_space_limit + fi collect_node_ls_heap collect_node_ls_pipeline_worker_count collect_node_ls_pipeline_batch_size collect_node_ls_input - collect_es_space_limit else - NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE + if [[ ! $is_receiver ]]; then + NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE + fi NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE LSPIPELINEWORKERS=$num_cpu_cores LSPIPELINEBATCH=125 @@ -727,6 +737,12 @@ echo "1" > /root/accept_changes fi fi + if [[ $is_node || $is_receiver ]]; then + set_progress_str 19 'Generating logstash pillar' + logstash_pillar >> $setup_log 2>&1 + fi + + if [[ $is_minion ]]; then set_progress_str 20 'Accepting Salt key on manager' retry 20 10 accept_salt_key_remote "going to be accepted" >> $setup_log 2>&1