From 70e4ce3e9894925584737fe7ffb00c13b5c52e8b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Jul 2020 14:38:51 -0400 Subject: [PATCH 1/8] Add batch to output --- .../config/so/9999_output_redis.conf.jinja | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index f176e0b94..594917ad6 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,26 +1,14 @@ -{%- if salt['grains.get']('role') == 'so-master' %} -{% set master = salt['pillar.get']('static:masterip', '') %} -{%- set nodetype = 'master' %} -{% elif grains.role == 'so-heavynode' %} -{% set master = salt['pillar.get']('node:mainip', '') %} -{%- set nodetype = salt['pillar.get']('node:node_type', 'search') %} -{%- else %} -{%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %} -{% set master = salt['pillar.get']('static:masterip', '') %} -{%- endif %} - +{% set MASTER = salt['pillar.get']('static:masterip', '') %} +{% set BATCH = salt['pillar.get']('logstash:batch_size', 500) %} output { redis { - host => '{{ master }}' + host => '{{ MASTER }}' data_type => 'list' - {%- if nodetype == 'parser' %} - key => 'logstash:parsed' - {%- else %} key => 'logstash:unparsed' - {%- endif %} congestion_interval => 1 congestion_threshold => 50000000 - # batch_events => 500 + batch ==> True + batch_events => {{ BATCH }} } } From f580da5d563524aed19fbc4695ae945467ba739a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Jul 2020 14:45:54 -0400 Subject: [PATCH 2/8] Update 9999_output_redis.conf.jinja --- salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 594917ad6..2a5ede0f4 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,5 +1,5 @@ {% set MASTER = salt['pillar.get']('static:masterip', '') %} -{% set BATCH = salt['pillar.get']('logstash:batch_size', 500) %} +{% set BATCH = salt['pillar.get']('logstash:batch_size', 125) %} output { redis { From 5580f05daf37a829ba226adb71d262d89d43449b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Jul 2020 15:07:00 -0400 Subject: [PATCH 3/8] Add logstash pillar --- setup/so-functions | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fdfcf37eb..9f2f7edfd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -873,13 +873,11 @@ master_pillar() { " esheap: $ES_HEAP_SIZE"\ " esclustername: {{ grains.host }}"\ " freq: 0"\ - " domainstats: 0" >> "$pillar_file" + " domainstats: 0" > "$pillar_file" + if [ "$install_type" = 'EVAL' ] || [ "$install_type" = 'HELIXSENSOR' ] || [ "$install_type" = 'MASTERSEARCH' ] || [ "$install_type" = 'STANDALONE' ]; then printf '%s\n'\ - " ls_pipeline_batch_size: 125"\ - " ls_input_threads: 1"\ - " ls_batch_count: 125"\ " mtu: $MTU" >> "$pillar_file" fi @@ -911,6 +909,11 @@ master_pillar() { " playbook: $PLAYBOOK"\ " url_base: $REDIRECTIT"\ ""\ + "logstash:"\ + " ls_pipeline_batch_size: 125"\ + " ls_input_threads: 1"\ + " ls_batch_count: 125"\ + ""\ "kratos:" >> "$pillar_file" printf '%s\n'\ @@ -1007,8 +1010,6 @@ node_pillar() { " esheap: $NODE_ES_HEAP_SIZE"\ " esclustername: {{ grains.host }}"\ " lsheap: $NODE_LS_HEAP_SIZE"\ - " ls_pipeline_workers: $LSPIPELINEWORKERS"\ - " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ " ls_input_threads: $LSINPUTTHREADS"\ " ls_batch_count: $LSINPUTBATCHCOUNT"\ " es_shard_count: $SHARDCOUNT"\ @@ -1016,6 +1017,10 @@ node_pillar() { " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ " cur_close_days: $CURCLOSEDAYS"\ + "logstash:"\ + " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ + " ls_input_threads: $LSINPUTTHREADS"\ + " ls_batch_count: $LSINPUTBATCHCOUNT"\ "" >> "$pillar_file" printf '%s\n' '----' >> "$setup_log" 2>&1 From ab42126d8e96bdcf8cf4792ff11b992f28b8351a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Jul 2020 15:25:35 -0400 Subject: [PATCH 4/8] Add logstash_settings pillar --- salt/logstash/init.sls | 9 +-------- .../pipelines/config/so/0900_input_redis.conf.jinja | 10 ++++++---- .../config/so/9999_output_redis.conf.jinja | 2 +- salt/redis/init.sls | 1 - setup/so-functions | 13 +++++-------- 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 1118b6807..b42ea4546 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -23,19 +23,12 @@ {% endif %} # Logstash Section - Decide which pillar to use -{% if grains['role'] == 'so-sensor' %} - {% set lsheap = salt['pillar.get']('sensor:lsheap', '') %} - {% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %} -{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} - {% set lsheap = salt['pillar.get']('node:lsheap', '') %} - {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} +{% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %} {% elif grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} - {% set lsheap = salt['pillar.get']('master:lsheap', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} {% elif grains['role'] == 'so-helix' %} - {% set lsheap = salt['pillar.get']('master:lsheap', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index ede940367..dc2f7c4ce 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,14 +1,16 @@ {%- if grains.role == 'so-heavynode' %} -{%- set master = salt['pillar.get']('node:mainip', '') %} +{%- set MASTER = salt['pillar.get']('node:mainip', '') %} {%- else %} -{%- set master = salt['pillar.get']('static:masterip', '') %} +{%- set MASTER = salt['pillar.get']('static:masterip', '') %} {% endif -%} +{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} + input { redis { - host => '{{ master }}' + host => '{{ MASTER }}' data_type => 'list' key => 'logstash:unparsed' type => 'redis-input' - # threads => 1 + threads => {{ THREADS }} } } diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 2a5ede0f4..5bf2512d5 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,5 +1,5 @@ {% set MASTER = salt['pillar.get']('static:masterip', '') %} -{% set BATCH = salt['pillar.get']('logstash:batch_size', 125) %} +{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} output { redis { diff --git a/salt/redis/init.sls b/salt/redis/init.sls index 3b2f662f3..5db53957c 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -12,7 +12,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set lsaccessip = salt['pillar.get']('master:lsaccessip', '') %} {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} {% set MASTER = salt['grains.get']('master') %} diff --git a/setup/so-functions b/setup/so-functions index 9f2f7edfd..0c8296c02 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -894,10 +894,7 @@ master_pillar() { esac printf '%s\n'\ - " lsheap: $LS_HEAP_SIZE"\ - " lsaccessip: 127.0.0.1"\ " elastalert: 1"\ - " ls_pipeline_workers: $num_cpu_cores"\ " nids_rules: $RULESETUP"\ " oinkcode: $OINKCODE"\ " es_port: $node_es_port"\ @@ -909,10 +906,12 @@ master_pillar() { " playbook: $PLAYBOOK"\ " url_base: $REDIRECTIT"\ ""\ - "logstash:"\ + "logstash_settings:"\ " ls_pipeline_batch_size: 125"\ " ls_input_threads: 1"\ " ls_batch_count: 125"\ + " lsheap: $LS_HEAP_SIZE"\ + " ls_pipeline_workers: $num_cpu_cores"\ ""\ "kratos:" >> "$pillar_file" @@ -1009,18 +1008,16 @@ node_pillar() { " mainint: $MNIC"\ " esheap: $NODE_ES_HEAP_SIZE"\ " esclustername: {{ grains.host }}"\ - " lsheap: $NODE_LS_HEAP_SIZE"\ - " ls_input_threads: $LSINPUTTHREADS"\ - " ls_batch_count: $LSINPUTBATCHCOUNT"\ " es_shard_count: $SHARDCOUNT"\ " node_type: $NODETYPE"\ " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ " cur_close_days: $CURCLOSEDAYS"\ - "logstash:"\ + "logstash_settings:"\ " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ " ls_input_threads: $LSINPUTTHREADS"\ " ls_batch_count: $LSINPUTBATCHCOUNT"\ + " lsheap: $NODE_LS_HEAP_SIZE"\ "" >> "$pillar_file" printf '%s\n' '----' >> "$setup_log" 2>&1 From 087080d583f9c159c28ff42528fb020068505201 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 6 Jul 2020 13:16:40 -0400 Subject: [PATCH 5/8] Add logix for logstash_settings --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 0c8296c02..81f45cb7b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1013,11 +1013,13 @@ node_pillar() { " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ " cur_close_days: $CURCLOSEDAYS"\ + if [ "$install_type" != 'EVAL' ] || [ "$install_type" != 'HELIXSENSOR' ] || [ "$install_type" != 'MASTERSEARCH' ] || [ "$install_type" != 'STANDALONE' ]; then "logstash_settings:"\ " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ " ls_input_threads: $LSINPUTTHREADS"\ " ls_batch_count: $LSINPUTBATCHCOUNT"\ " lsheap: $NODE_LS_HEAP_SIZE"\ + fi "" >> "$pillar_file" printf '%s\n' '----' >> "$setup_log" 2>&1 From 1016315196d773e8dba1d0a00ee551f2555c8592 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 6 Jul 2020 13:58:38 -0400 Subject: [PATCH 6/8] Change Logic for logstash --- setup/so-functions | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 81f45cb7b..8af0208ea 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1013,14 +1013,17 @@ node_pillar() { " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ " cur_close_days: $CURCLOSEDAYS"\ - if [ "$install_type" != 'EVAL' ] || [ "$install_type" != 'HELIXSENSOR' ] || [ "$install_type" != 'MASTERSEARCH' ] || [ "$install_type" != 'STANDALONE' ]; then + "" >> "$pillar_file" + + if [ "$install_type" != 'EVAL' ] && [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'MASTERSEARCH' ] && [ "$install_type" != 'STANDALONE' ]; then + printf '%s\n'\ "logstash_settings:"\ " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ " ls_input_threads: $LSINPUTTHREADS"\ " ls_batch_count: $LSINPUTBATCHCOUNT"\ " lsheap: $NODE_LS_HEAP_SIZE"\ - fi "" >> "$pillar_file" + fi printf '%s\n' '----' >> "$setup_log" 2>&1 cat "$pillar_file" >> "$setup_log" 2>&1 From be3390a796436b0b84a226bf5e4b1827437f5cc1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 6 Jul 2020 15:53:21 -0400 Subject: [PATCH 7/8] Fix Logstash state --- salt/logstash/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index b42ea4546..e2494e57a 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -24,7 +24,7 @@ # Logstash Section - Decide which pillar to use {% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %} -{% elif grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} +{% if grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} From f05e366d492f11f024877f2b4cbf38f90fa95aaf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 6 Jul 2020 15:56:55 -0400 Subject: [PATCH 8/8] Fix salt upgrade script --- salt/common/tools/sbin/so-saltstack-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-saltstack-update b/salt/common/tools/sbin/so-saltstack-update index ce323025d..59c497487 100644 --- a/salt/common/tools/sbin/so-saltstack-update +++ b/salt/common/tools/sbin/so-saltstack-update @@ -30,7 +30,7 @@ clone_to_tmp() { copy_new_files() { # Copy new files over to the salt dir - cd /tmp/sogh/securityonion-saltstack + cd /tmp/sogh/securityonion git checkout $BRANCH rsync -a --exclude-from 'exclude-list.txt' salt $default_salt_dir/ rsync -a --exclude-from 'exclude-list.txt' pillar $default_salt_dir/