Merge pull request #934 from Security-Onion-Solutions/issue/142

Issue/142

salt/common/tools/sbin/so-saltstack-update

@@ -30,7 +30,7 @@ clone_to_tmp() {
 copy_new_files() {
 
   # Copy new files over to the salt dir
-  cd /tmp/sogh/securityonion-saltstack
+  cd /tmp/sogh/securityonion
   git checkout $BRANCH
   rsync -a --exclude-from 'exclude-list.txt' salt $default_salt_dir/
   rsync -a --exclude-from 'exclude-list.txt' pillar $default_salt_dir/

salt/logstash/init.sls

@@ -23,19 +23,12 @@
 {% endif %}
 
 # Logstash Section - Decide which pillar to use
-{% if grains['role'] == 'so-sensor' %}
+{% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %}
-  {% set lsheap = salt['pillar.get']('sensor:lsheap', '') %}
+{% if grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %}
-  {% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %}
-{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
-  {% set lsheap = salt['pillar.get']('node:lsheap', '') %}
-  {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %}
-{% elif grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %}
-  {% set lsheap = salt['pillar.get']('master:lsheap', '') %}
   {% set freq = salt['pillar.get']('master:freq', '0') %}
   {% set dstats = salt['pillar.get']('master:domainstats', '0') %}
   {% set nodetype = salt['grains.get']('role', '')  %}
 {% elif grains['role'] == 'so-helix' %}
-  {% set lsheap = salt['pillar.get']('master:lsheap', '') %}
   {% set freq = salt['pillar.get']('master:freq', '0') %}
   {% set dstats = salt['pillar.get']('master:domainstats', '0') %}
   {% set nodetype = salt['grains.get']('role', '')  %}

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,14 +1,16 @@
 {%- if grains.role == 'so-heavynode' %}
-{%- set master = salt['pillar.get']('node:mainip', '') %}
+{%- set MASTER = salt['pillar.get']('node:mainip', '') %}
 {%- else %}
-{%- set master = salt['pillar.get']('static:masterip', '') %}
+{%- set MASTER = salt['pillar.get']('static:masterip', '') %}
 {% endif -%}
+{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
+
 input {
 	redis {
-		host => '{{ master }}'
+		host => '{{ MASTER }}'
 		data_type => 'list'
 		key => 'logstash:unparsed'
 		type => 'redis-input'
-		# threads => 1
+		threads => {{ THREADS }}
 	}
 }

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,26 +1,14 @@
-{%- if salt['grains.get']('role') == 'so-master' %}
+{% set MASTER = salt['pillar.get']('static:masterip', '') %}
-{% set master = salt['pillar.get']('static:masterip', '') %}
+{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
-{%- set nodetype = 'master' %}
-{% elif grains.role == 'so-heavynode' %}
-{% set master = salt['pillar.get']('node:mainip', '') %}
-{%- set nodetype = salt['pillar.get']('node:node_type', 'search') %}
-{%- else %}
-{%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %}
-{% set master = salt['pillar.get']('static:masterip', '') %}
-{%- endif %}
-
 
 output {
 	redis {
-		host => '{{ master }}'
+		host => '{{ MASTER }}'
 		data_type => 'list'
-		{%- if nodetype == 'parser' %}
-		key => 'logstash:parsed'
-		{%- else %}
 		key => 'logstash:unparsed'
-		{%- endif %}
 		congestion_interval => 1
 		congestion_threshold => 50000000
-		# batch_events => 500
+		batch ==> True
+		batch_events => {{ BATCH }}
 	}
 }

salt/redis/init.sls

@@ -12,7 +12,6 @@
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-{% set lsaccessip = salt['pillar.get']('master:lsaccessip', '') %}
 {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %}
 {% set MASTER = salt['grains.get']('master') %}
 

setup/so-functions

@@ -923,13 +923,11 @@ master_pillar() {
 		"  esheap: $ES_HEAP_SIZE"\
 		"  esclustername: {{ grains.host }}"\
 		"  freq: 0"\
-		"  domainstats: 0" >> "$pillar_file"
+		"  domainstats: 0" > "$pillar_file"
+		
 
 	if [ "$install_type" = 'EVAL' ] || [ "$install_type" = 'HELIXSENSOR' ] || [ "$install_type" = 'MASTERSEARCH' ] || [ "$install_type" = 'STANDALONE' ]; then
 		printf '%s\n'\
-			"  ls_pipeline_batch_size: 125"\
-			"  ls_input_threads: 1"\
-			"  ls_batch_count: 125"\
 			"  mtu: $MTU" >> "$pillar_file"
 	fi
 
@@ -946,10 +944,7 @@ master_pillar() {
 	esac
 
 	printf '%s\n'\
-		"  lsheap: $LS_HEAP_SIZE"\
-		"  lsaccessip: 127.0.0.1"\
 		"  elastalert: 1"\
-		"  ls_pipeline_workers: $num_cpu_cores"\
 		"  nids_rules: $RULESETUP"\
 		"  oinkcode: $OINKCODE"\
 		"  es_port: $node_es_port"\
@@ -961,6 +956,13 @@ master_pillar() {
 		"  playbook: $PLAYBOOK"\
 		"  url_base: $REDIRECTIT"\
 		""\
+		"logstash_settings:"\
+		"  ls_pipeline_batch_size: 125"\
+		"  ls_input_threads: 1"\
+		"  ls_batch_count: 125"\
+		"  lsheap: $LS_HEAP_SIZE"\
+		"  ls_pipeline_workers: $num_cpu_cores"\
+		""\
 		"kratos:" >> "$pillar_file"
 
 	printf '%s\n'\
@@ -1056,11 +1058,6 @@ node_pillar() {
 		"  mainint: $MNIC"\
 		"  esheap: $NODE_ES_HEAP_SIZE"\
 		"  esclustername: {{ grains.host }}"\
-		"  lsheap: $NODE_LS_HEAP_SIZE"\
-		"  ls_pipeline_workers: $LSPIPELINEWORKERS"\
-		"  ls_pipeline_batch_size: $LSPIPELINEBATCH"\
-		"  ls_input_threads: $LSINPUTTHREADS"\
-		"  ls_batch_count: $LSINPUTBATCHCOUNT"\
 		"  es_shard_count: $SHARDCOUNT"\
 		"  node_type: $NODETYPE"\
 		"  es_port: $node_es_port"\
@@ -1068,6 +1065,16 @@ node_pillar() {
 		"  cur_close_days: $CURCLOSEDAYS"\
 		"" >> "$pillar_file"
 
+	if [ "$install_type" != 'EVAL' ] && [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'MASTERSEARCH' ] && [ "$install_type" != 'STANDALONE' ]; then
+	printf '%s\n'\
+		"logstash_settings:"\
+		"  ls_pipeline_batch_size: $LSPIPELINEBATCH"\
+		"  ls_input_threads: $LSINPUTTHREADS"\
+		"  ls_batch_count: $LSINPUTBATCHCOUNT"\
+		"  lsheap: $NODE_LS_HEAP_SIZE"\
+		"" >> "$pillar_file"
+	fi
+
 	printf '%s\n'  '----' >> "$setup_log" 2>&1
 	cat "$pillar_file" >> "$setup_log" 2>&1
 }