Add logstash_settings pillar

salt/logstash/init.sls

@@ -23,19 +23,12 @@
 {% endif %}
 
 # Logstash Section - Decide which pillar to use
-{% if grains['role'] == 'so-sensor' %}
-  {% set lsheap = salt['pillar.get']('sensor:lsheap', '') %}
-  {% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %}
-{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
-  {% set lsheap = salt['pillar.get']('node:lsheap', '') %}
-  {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %}
+{% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %}
 {% elif grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %}
-  {% set lsheap = salt['pillar.get']('master:lsheap', '') %}
   {% set freq = salt['pillar.get']('master:freq', '0') %}
   {% set dstats = salt['pillar.get']('master:domainstats', '0') %}
   {% set nodetype = salt['grains.get']('role', '')  %}
 {% elif grains['role'] == 'so-helix' %}
-  {% set lsheap = salt['pillar.get']('master:lsheap', '') %}
   {% set freq = salt['pillar.get']('master:freq', '0') %}
   {% set dstats = salt['pillar.get']('master:domainstats', '0') %}
   {% set nodetype = salt['grains.get']('role', '')  %}

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,14 +1,16 @@
 {%- if grains.role == 'so-heavynode' %}
-{%- set master = salt['pillar.get']('node:mainip', '') %}
+{%- set MASTER = salt['pillar.get']('node:mainip', '') %}
 {%- else %}
-{%- set master = salt['pillar.get']('static:masterip', '') %}
+{%- set MASTER = salt['pillar.get']('static:masterip', '') %}
 {% endif -%}
+{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
+
 input {
 	redis {
-		host => '{{ master }}'
+		host => '{{ MASTER }}'
 		data_type => 'list'
 		key => 'logstash:unparsed'
 		type => 'redis-input'
-		# threads => 1
+		threads => {{ THREADS }}
 	}
 }

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,5 +1,5 @@
 {% set MASTER = salt['pillar.get']('static:masterip', '') %}
-{% set BATCH = salt['pillar.get']('logstash:batch_size', 125) %}
+{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
 
 output {
 	redis {

salt/redis/init.sls

@@ -12,7 +12,6 @@
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-{% set lsaccessip = salt['pillar.get']('master:lsaccessip', '') %}
 {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %}
 {% set MASTER = salt['grains.get']('master') %}
 

setup/so-functions

@@ -894,10 +894,7 @@ master_pillar() {
 	esac
 
 	printf '%s\n'\
-		"  lsheap: $LS_HEAP_SIZE"\
-		"  lsaccessip: 127.0.0.1"\
 		"  elastalert: 1"\
-		"  ls_pipeline_workers: $num_cpu_cores"\
 		"  nids_rules: $RULESETUP"\
 		"  oinkcode: $OINKCODE"\
 		"  es_port: $node_es_port"\
@@ -909,10 +906,12 @@ master_pillar() {
 		"  playbook: $PLAYBOOK"\
 		"  url_base: $REDIRECTIT"\
 		""\
-		"logstash:"\
+		"logstash_settings:"\
 		"  ls_pipeline_batch_size: 125"\
 		"  ls_input_threads: 1"\
 		"  ls_batch_count: 125"\
+		"  lsheap: $LS_HEAP_SIZE"\
+		"  ls_pipeline_workers: $num_cpu_cores"\
 		""\
 		"kratos:" >> "$pillar_file"
 
@@ -1009,18 +1008,16 @@ node_pillar() {
 		"  mainint: $MNIC"\
 		"  esheap: $NODE_ES_HEAP_SIZE"\
 		"  esclustername: {{ grains.host }}"\
-		"  lsheap: $NODE_LS_HEAP_SIZE"\
-		"  ls_input_threads: $LSINPUTTHREADS"\
-		"  ls_batch_count: $LSINPUTBATCHCOUNT"\
 		"  es_shard_count: $SHARDCOUNT"\
 		"  node_type: $NODETYPE"\
 		"  es_port: $node_es_port"\
 		"  log_size_limit: $log_size_limit"\
 		"  cur_close_days: $CURCLOSEDAYS"\
-		"logstash:"\
+		"logstash_settings:"\
 		"  ls_pipeline_batch_size: $LSPIPELINEBATCH"\
 		"  ls_input_threads: $LSINPUTTHREADS"\
 		"  ls_batch_count: $LSINPUTBATCHCOUNT"\
+		"  lsheap: $NODE_LS_HEAP_SIZE"\
 		"" >> "$pillar_file"
 
 	printf '%s\n'  '----' >> "$setup_log" 2>&1