From 4946f32d88b4fca0083c369c81131b41402fa244 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 6 Jul 2021 14:14:58 -0400 Subject: [PATCH 1/7] Add extra_hosts entry for local instance when running as heavy node --- salt/logstash/init.sls | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index bfd08e4fe..86405fa79 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -145,7 +145,12 @@ so-logstash: - name: so-logstash - user: logstash - extra_hosts: + {% if grains.role in ['so-heavynode'] %} + {% set MANAGER = salt['grains.get']('nodename') %} + {% set MANAGERIP = salt['pillar.get']('sensor:mainip') %} + {% else %} - {{ MANAGER }}:{{ MANAGERIP }} + {% endif %} - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} - port_bindings: @@ -205,4 +210,4 @@ append_so-logstash_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From 10b1829830cc37b897fadf3d57468b49d57a6202 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 6 Jul 2021 14:16:34 -0400 Subject: [PATCH 2/7] Add conditional for heavynode --- salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index 60229422b..4b108d1b6 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,4 +1,8 @@ +{%- if grains.role in ['so-heavynode'] %} +{%- set MANAGER = salt['grains.get']('host') %} +{%- else %} {%- set MANAGER = salt['grains.get']('master') %} +{%- endif %} {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} From 2e91f2733626a4befc5e55364a4c318d1ad8845d Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 6 Jul 2021 14:17:49 -0400 Subject: [PATCH 3/7] Add conditional for heavynode --- .../logstash/pipelines/config/so/9999_output_redis.conf.jinja | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 626ed62c3..a70e309e4 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,4 +1,8 @@ +{%- if grains.role in ['so-heavynode'] %} +{%- set MANAGER = salt['grains.get']('host') %} +{%- else %} {%- set MANAGER = salt['grains.get']('master') %} +{%- endif %} {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} output { redis { From 2f3f04e4ca8363ec7f7de3ca8bcc11ca4ca2cef1 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 6 Jul 2021 14:18:39 -0400 Subject: [PATCH 4/7] Change from nodename to host --- salt/logstash/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 86405fa79..581d1349a 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -146,7 +146,7 @@ so-logstash: - user: logstash - extra_hosts: {% if grains.role in ['so-heavynode'] %} - {% set MANAGER = salt['grains.get']('nodename') %} + {% set MANAGER = salt['grains.get']('host') %} {% set MANAGERIP = salt['pillar.get']('sensor:mainip') %} {% else %} - {{ MANAGER }}:{{ MANAGERIP }} From ce0b0649723e7f85110f9a8bed4da185fc5a264f Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 6 Jul 2021 14:21:29 -0400 Subject: [PATCH 5/7] Add conditional for heavynode for redis and elasticsearch --- salt/ssl/init.sls | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 5786437f6..d885a2721 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -9,6 +9,9 @@ {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} +{% if grains.role in ['so-heavynode'] %} +{% set heavynode = salt['grains.get']('host') %} +{% endif %} {% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'import', 'helixsensor'] %} {% set trusttheca_text = salt['cp.get_file_str']('/etc/pki/ca.crt')|replace('\n', '') %} @@ -103,7 +106,11 @@ influxkeyperms: # Create a cert for Redis encryption /etc/pki/redis.key: x509.private_key_managed: + {% if grains.role in ['so-heavynode'] %} + - CN: {{ heavynode }} + {% else %} - CN: {{ manager }} + {% endif %} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -123,7 +130,11 @@ influxkeyperms: - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/redis.key + {% if grains.role in ['so-heavynode'] %} + - CN: {{ heavynode }} + {% else %} - CN: {{ manager }} + {% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -315,7 +326,11 @@ miniokeyperms: # Create a cert for elasticsearch /etc/pki/elasticsearch.key: x509.private_key_managed: + {% if grains.role in ['so-heavynode'] %} + - CN: {{ heavynode }} + {% else %} - CN: {{ manager }} + {% endif %} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -335,7 +350,11 @@ miniokeyperms: - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/elasticsearch.key + {% if grains.role in ['so-heavynode'] %} + - CN: {{ heavynode }} + {% else %} - CN: {{ manager }} + {% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -677,4 +696,4 @@ elastickeyperms: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From 0627ca2fc2e245e0b073062f2c1664067543503d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Jul 2021 15:32:39 -0400 Subject: [PATCH 6/7] use heavynode hostname for certs if heavynode. changes to logstash pipeline for redis if heavynode --- salt/logstash/init.sls | 15 ++-- .../config/so/0900_input_redis.conf.jinja | 11 ++- .../config/so/9999_output_redis.conf.jinja | 8 +-- salt/ssl/init.sls | 70 +++++++------------ 4 files changed, 42 insertions(+), 62 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 581d1349a..cd6a8918c 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -36,6 +36,14 @@ {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% if grains.role in ['so-heavynode'] %} + {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %} + {% set EXTRAHOSTIP = salt['pillar.get']('sensor:mainip') %} +{% else %} + {% set EXTRAHOSTHOSTNAME = MANAGER %} + {% set EXTRAHOSTIP = MANAGERIP %} +{% endif %} + include: - elasticsearch @@ -145,12 +153,7 @@ so-logstash: - name: so-logstash - user: logstash - extra_hosts: - {% if grains.role in ['so-heavynode'] %} - {% set MANAGER = salt['grains.get']('host') %} - {% set MANAGERIP = salt['pillar.get']('sensor:mainip') %} - {% else %} - - {{ MANAGER }}:{{ MANAGERIP }} - {% endif %} + - {{ EXTRAHOSTHOSTNAME }}:{{ EXTRAHOSTIP }} - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} - port_bindings: diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index 4b108d1b6..35f77c5a0 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,14 +1,13 @@ {%- if grains.role in ['so-heavynode'] %} -{%- set MANAGER = salt['grains.get']('host') %} + {%- set HOST = salt['grains.get']('host') %} {%- else %} -{%- set MANAGER = salt['grains.get']('master') %} + {%- set HOST = salt['grains.get']('master') %} {%- endif %} -{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} - + {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} +{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} input { redis { - host => '{{ MANAGER }}' + host => '{{ HOST }}' port => 9696 ssl => true data_type => 'list' diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index a70e309e4..6b9c62e2f 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,12 +1,12 @@ {%- if grains.role in ['so-heavynode'] %} -{%- set MANAGER = salt['grains.get']('host') %} + {%- set HOST = salt['grains.get']('host') %} {%- else %} -{%- set MANAGER = salt['grains.get']('master') %} + {%- set HOST = salt['grains.get']('master') %} {%- endif %} -{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} +{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} output { redis { - host => '{{ MANAGER }}' + host => '{{ HOST }}' port => 6379 data_type => 'list' key => 'logstash:unparsed' diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index d885a2721..aaa650f97 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -10,7 +10,9 @@ {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} {% if grains.role in ['so-heavynode'] %} -{% set heavynode = salt['grains.get']('host') %} + {% set COMMONNAME = salt['grains.get']('host') %} +{% else %} + {% set COMMONNAME = manager %} {% endif %} {% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'import', 'helixsensor'] %} @@ -60,7 +62,7 @@ removeesp12dir: /etc/pki/influxdb.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -81,7 +83,7 @@ removeesp12dir: - ca_server: {{ ca_server }} - signing_policy: influxdb - public_key: /etc/pki/influxdb.key - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - subjectAltName: DNS:{{ HOSTNAME }} - days_remaining: 0 - days_valid: 820 @@ -106,11 +108,7 @@ influxkeyperms: # Create a cert for Redis encryption /etc/pki/redis.key: x509.private_key_managed: - {% if grains.role in ['so-heavynode'] %} - - CN: {{ heavynode }} - {% else %} - - CN: {{ manager }} - {% endif %} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -130,11 +128,7 @@ influxkeyperms: - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/redis.key - {% if grains.role in ['so-heavynode'] %} - - CN: {{ heavynode }} - {% else %} - - CN: {{ manager }} - {% endif %} + - CN: {{ COMMONNAME }} - days_remaining: 0 - days_valid: 820 - backup: True @@ -158,7 +152,7 @@ rediskeyperms: {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %} /etc/pki/filebeat.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -179,11 +173,7 @@ rediskeyperms: - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /etc/pki/filebeat.key -{% if grains.role == 'so-heavynode' %} - - CN: {{grains.host}} -{% else %} - - CN: {{manager}} -{% endif %} + - CN: {{ COMMONNAME }} - days_remaining: 0 - days_valid: 820 - backup: True @@ -239,7 +229,7 @@ fbcrtlink: /etc/pki/registry.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -260,7 +250,7 @@ fbcrtlink: - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/registry.key - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - days_remaining: 0 - days_valid: 820 - backup: True @@ -282,7 +272,7 @@ regkeyperms: /etc/pki/minio.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -303,7 +293,7 @@ regkeyperms: - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/minio.key - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - days_remaining: 0 - days_valid: 820 - backup: True @@ -326,11 +316,7 @@ miniokeyperms: # Create a cert for elasticsearch /etc/pki/elasticsearch.key: x509.private_key_managed: - {% if grains.role in ['so-heavynode'] %} - - CN: {{ heavynode }} - {% else %} - - CN: {{ manager }} - {% endif %} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -350,11 +336,7 @@ miniokeyperms: - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/elasticsearch.key - {% if grains.role in ['so-heavynode'] %} - - CN: {{ heavynode }} - {% else %} - - CN: {{ manager }} - {% endif %} + - CN: {{ COMMONNAME }} - days_remaining: 0 - days_valid: 820 - backup: True @@ -387,7 +369,7 @@ elasticp12perms: /etc/pki/managerssl.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -408,7 +390,7 @@ elasticp12perms: - ca_server: {{ ca_server }} - signing_policy: managerssl - public_key: /etc/pki/managerssl.key - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }} {% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }} {% endif %} - days_remaining: 0 - days_valid: 820 @@ -432,7 +414,7 @@ msslkeyperms: # Create a private key and cert for OSQuery /etc/pki/fleet.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -450,7 +432,7 @@ msslkeyperms: /etc/pki/fleet.crt: x509.certificate_managed: - signing_private_key: /etc/pki/fleet.key - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - subjectAltName: DNS:{{ manager }},IP:{{ managerip }} - days_remaining: 0 - days_valid: 820 @@ -481,7 +463,7 @@ fbcertdir: /opt/so/conf/filebeat/etc/pki/filebeat.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -502,11 +484,7 @@ fbcertdir: - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key -{% if grains.role == 'so-heavynode' %} - - CN: {{grains.id}} -{% else %} - - CN: {{manager}} -{% endif %} + - CN: {{ COMMONNAME }} - days_remaining: 0 - days_valid: 820 - backup: True @@ -547,7 +525,7 @@ chownfilebeatp8: /etc/pki/managerssl.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -592,7 +570,7 @@ msslkeyperms: # Create a private key and cert for Fleet /etc/pki/fleet.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 @@ -637,7 +615,7 @@ fleetkeyperms: # Create a cert for elasticsearch /etc/pki/elasticsearch.key: x509.private_key_managed: - - CN: {{ manager }} + - CN: {{ COMMONNAME }} - bits: 4096 - days_remaining: 0 - days_valid: 820 From c1d61dc6248f4eef24154846a9100c7610c467d7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Jul 2021 15:54:15 -0400 Subject: [PATCH 7/7] add to HOTFIX file --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index f32bc0381..a87f12a32 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -ECSFIX +HEAVYNODE_SSL_LOGSTASH_REDIS_PIPELINES