extra_hosts and redis_input for logstash

salt/logstash/init.sls

@@ -36,13 +36,14 @@
   {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
   {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 
-  {% if grains.role in ['so-heavynode', 'so-receiver'] %}
+  {# if grains.role in ['so-heavynode', 'so-receiver'] %}
     {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %}
     {% set EXTRAHOSTIP = salt['grains.get']('ip_interfaces').get(salt['pillar.get']('host:mainint'))[0] %}
   {% else %}
     {% set EXTRAHOSTHOSTNAME = MANAGER %}
     {% set EXTRAHOSTIP = MANAGERIP %}
-  {% endif %}
+  {% endif #}
+  {% from 'logstash/map.jinja' import EXTRA_HOSTS with context %}
 
 include:
   - ssl
@@ -155,8 +156,7 @@ so-logstash:
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash
-    - extra_hosts:
-      - {{ EXTRAHOSTHOSTNAME }}:{{ EXTRAHOSTIP }}
+    - extra_hosts: {{ EXTRA_HOSTS }}
     - environment:
       - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
     - port_bindings:

salt/logstash/map.jinja

@@ -0,0 +1,16 @@
+{% set role = grains.role %}
+{% set EXTRA_HOSTS = [] %}
+{% set mainint = salt['pillar.get']('host:mainint') %}
+{% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
+{% if role in ['so-node'] %}
+  {% set node_data = salt['pillar.get']('logstash:nodes') %}
+  {% for node_type, node_details in node_data.items() | sort %}
+    {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
+      {% for hostname in node_data[node_type].keys() %}
+        {% do EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %}
+      {% endfor %}
+    {% endif %}
+  {% endfor %}
+{% else %}
+  {% do EXTRA_HOSTS.append({grains.host:localhostip}) %}
+{% endif %}

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,13 +1,12 @@
-{%- if grains.role in ['so-heavynode'] %}
-  {%- set HOST = salt['grains.get']('host') %}
-{%- else %}
-  {%- set HOST = salt['grains.get']('master') %}
-{%- endif %}
 {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
 {%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
+
+{%- from 'logstash/map.jinja' import EXTRA_HOSTS with context %}
+
+{%- for host in EXTRA_HOSTS.keys %}
 input {
 	redis {
-		host => '{{ HOST }}'
+		host => '{{ host }}'
 		port => 9696
 		ssl => true
 		data_type => 'list'
@@ -17,3 +16,4 @@ input {
 		batch_count => {{ BATCH }}
 	}
 }
+{%- endfor %}