diff --git a/salt/common/tools/sbin/so-saltstack-update b/salt/common/tools/sbin/so-saltstack-update index ce323025d..59c497487 100644 --- a/salt/common/tools/sbin/so-saltstack-update +++ b/salt/common/tools/sbin/so-saltstack-update @@ -30,7 +30,7 @@ clone_to_tmp() { copy_new_files() { # Copy new files over to the salt dir - cd /tmp/sogh/securityonion-saltstack + cd /tmp/sogh/securityonion git checkout $BRANCH rsync -a --exclude-from 'exclude-list.txt' salt $default_salt_dir/ rsync -a --exclude-from 'exclude-list.txt' pillar $default_salt_dir/ diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 1118b6807..e2494e57a 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -23,19 +23,12 @@ {% endif %} # Logstash Section - Decide which pillar to use -{% if grains['role'] == 'so-sensor' %} - {% set lsheap = salt['pillar.get']('sensor:lsheap', '') %} - {% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %} -{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} - {% set lsheap = salt['pillar.get']('node:lsheap', '') %} - {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} -{% elif grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} - {% set lsheap = salt['pillar.get']('master:lsheap', '') %} +{% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %} +{% if grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} {% elif grains['role'] == 'so-helix' %} - {% set lsheap = salt['pillar.get']('master:lsheap', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index ede940367..dc2f7c4ce 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,14 +1,16 @@ {%- if grains.role == 'so-heavynode' %} -{%- set master = salt['pillar.get']('node:mainip', '') %} +{%- set MASTER = salt['pillar.get']('node:mainip', '') %} {%- else %} -{%- set master = salt['pillar.get']('static:masterip', '') %} +{%- set MASTER = salt['pillar.get']('static:masterip', '') %} {% endif -%} +{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} + input { redis { - host => '{{ master }}' + host => '{{ MASTER }}' data_type => 'list' key => 'logstash:unparsed' type => 'redis-input' - # threads => 1 + threads => {{ THREADS }} } } diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index f176e0b94..5bf2512d5 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,26 +1,14 @@ -{%- if salt['grains.get']('role') == 'so-master' %} -{% set master = salt['pillar.get']('static:masterip', '') %} -{%- set nodetype = 'master' %} -{% elif grains.role == 'so-heavynode' %} -{% set master = salt['pillar.get']('node:mainip', '') %} -{%- set nodetype = salt['pillar.get']('node:node_type', 'search') %} -{%- else %} -{%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %} -{% set master = salt['pillar.get']('static:masterip', '') %} -{%- endif %} - +{% set MASTER = salt['pillar.get']('static:masterip', '') %} +{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} output { redis { - host => '{{ master }}' + host => '{{ MASTER }}' data_type => 'list' - {%- if nodetype == 'parser' %} - key => 'logstash:parsed' - {%- else %} key => 'logstash:unparsed' - {%- endif %} congestion_interval => 1 congestion_threshold => 50000000 - # batch_events => 500 + batch ==> True + batch_events => {{ BATCH }} } } diff --git a/salt/redis/init.sls b/salt/redis/init.sls index 3b2f662f3..5db53957c 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -12,7 +12,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set lsaccessip = salt['pillar.get']('master:lsaccessip', '') %} {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} {% set MASTER = salt['grains.get']('master') %} diff --git a/setup/so-functions b/setup/so-functions index 00b00f3ab..cd81074f2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -923,13 +923,11 @@ master_pillar() { " esheap: $ES_HEAP_SIZE"\ " esclustername: {{ grains.host }}"\ " freq: 0"\ - " domainstats: 0" >> "$pillar_file" + " domainstats: 0" > "$pillar_file" + if [ "$install_type" = 'EVAL' ] || [ "$install_type" = 'HELIXSENSOR' ] || [ "$install_type" = 'MASTERSEARCH' ] || [ "$install_type" = 'STANDALONE' ]; then printf '%s\n'\ - " ls_pipeline_batch_size: 125"\ - " ls_input_threads: 1"\ - " ls_batch_count: 125"\ " mtu: $MTU" >> "$pillar_file" fi @@ -946,10 +944,7 @@ master_pillar() { esac printf '%s\n'\ - " lsheap: $LS_HEAP_SIZE"\ - " lsaccessip: 127.0.0.1"\ " elastalert: 1"\ - " ls_pipeline_workers: $num_cpu_cores"\ " nids_rules: $RULESETUP"\ " oinkcode: $OINKCODE"\ " es_port: $node_es_port"\ @@ -961,6 +956,13 @@ master_pillar() { " playbook: $PLAYBOOK"\ " url_base: $REDIRECTIT"\ ""\ + "logstash_settings:"\ + " ls_pipeline_batch_size: 125"\ + " ls_input_threads: 1"\ + " ls_batch_count: 125"\ + " lsheap: $LS_HEAP_SIZE"\ + " ls_pipeline_workers: $num_cpu_cores"\ + ""\ "kratos:" >> "$pillar_file" printf '%s\n'\ @@ -1056,11 +1058,6 @@ node_pillar() { " mainint: $MNIC"\ " esheap: $NODE_ES_HEAP_SIZE"\ " esclustername: {{ grains.host }}"\ - " lsheap: $NODE_LS_HEAP_SIZE"\ - " ls_pipeline_workers: $LSPIPELINEWORKERS"\ - " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ - " ls_input_threads: $LSINPUTTHREADS"\ - " ls_batch_count: $LSINPUTBATCHCOUNT"\ " es_shard_count: $SHARDCOUNT"\ " node_type: $NODETYPE"\ " es_port: $node_es_port"\ @@ -1068,6 +1065,16 @@ node_pillar() { " cur_close_days: $CURCLOSEDAYS"\ "" >> "$pillar_file" + if [ "$install_type" != 'EVAL' ] && [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'MASTERSEARCH' ] && [ "$install_type" != 'STANDALONE' ]; then + printf '%s\n'\ + "logstash_settings:"\ + " ls_pipeline_batch_size: $LSPIPELINEBATCH"\ + " ls_input_threads: $LSINPUTTHREADS"\ + " ls_batch_count: $LSINPUTBATCHCOUNT"\ + " lsheap: $NODE_LS_HEAP_SIZE"\ + "" >> "$pillar_file" + fi + printf '%s\n' '----' >> "$setup_log" 2>&1 cat "$pillar_file" >> "$setup_log" 2>&1 }