From a641346c02f433e5d7863e180908889d1f553264 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 May 2022 17:28:19 -0400 Subject: [PATCH 1/3] prevent nodes with logstash:dmz:true from being added to logstash:nodes pillar --- pillar/logstash/nodes.sls | 2 +- salt/podman/init.sls | 51 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 salt/podman/init.sls diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index 18c4b39bf..92272e7d8 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -2,7 +2,7 @@ {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ', + tgt='( G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ) and ( not I@logstash:dmz:true or not I@logstash:dmz:True )', fun='network.ip_addrs', tgt_type='compound') | dictsort() %} diff --git a/salt/podman/init.sls b/salt/podman/init.sls new file mode 100644 index 000000000..f00a21074 --- /dev/null +++ b/salt/podman/init.sls @@ -0,0 +1,51 @@ +Pip pkg: + pkg.installed: + - name: python3-pip + +Podman pkg: + pkg.installed: + - name: podman + +Podman service: + file.managed: + - name: /usr/lib/systemd/system/podman.service + - source: salt://podman/podman.service + +Podman socket: + file.managed: + - name: /usr/lib/systemd/system/podman.socket + - source: salt://podman/podman.socket + service.running: + - name: podman.socket + - enable: true + +Docker socket: + file.symlink: + - name: /var/run/docker.sock + - target: /var/run/podman/podman.sock + +Docker python: + pip.installed: + - bin_env: /usr/bin/pip3 + - reload_modules: true + - pkgs: + - certifi==2019.11.28 + - chardet==3.0.4 + - docker==4.2.1 + - idna==2.9 + # - requests==2.23.0 + - six==1.14.0 + - urllib3==1.25.8 + - websocket-client==0.57.0 + +podman_docker_symlink: + file.symlink: + - name: /bin/docker + - target: /usr/bin/podman + +restart_salt_minion: + cmd.run: + - name: 'salt-call service.restart salt-minion' + - bg: true + - onchanges: + - pip: Docker python From d8abc0a19507ff0ea913a7ff32bf0ec6ecd9dc7a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 11 May 2022 11:51:18 -0400 Subject: [PATCH 2/3] if in dmz_nodes dont add to filebeta --- pillar/logstash/nodes.sls | 2 +- salt/filebeat/etc/filebeat.yml | 11 ++++++++++- salt/logstash/dmz_nodes.yaml | 9 +++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 salt/logstash/dmz_nodes.yaml diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index 92272e7d8..935574ff9 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -2,7 +2,7 @@ {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='( G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ) and ( not I@logstash:dmz:true or not I@logstash:dmz:True )', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix', fun='network.ip_addrs', tgt_type='compound') | dictsort() %} diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 62a45e9c4..d3b377bfb 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -443,6 +443,13 @@ output.logstash: # The Logstash hosts hosts: +{# dont let filebeat send to a node designated as dmz #} +{% import_yaml 'logstash/dmz_nodes.yaml' as dmz_nodes -%} +{% if dmz_nodes.logstash.dmz_nodes -%} +{% set dmz_nodes = dmz_nodes.logstash.dmz_nodes -%} +{% else -%} +{% set dmz_nodes = [] -%} +{% endif -%} {%- if grains.role in ['so-sensor', 'so-fleet', 'so-node', 'so-idh'] %} {%- set LOGSTASH = namespace() %} {%- set LOGSTASH.count = 0 %} @@ -451,8 +458,10 @@ output.logstash: {%- for node_type, node_details in node_data.items() | sort -%} {%- if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} {%- for hostname in node_data[node_type].keys() %} -{%- set LOGSTASH.count = LOGSTASH.count + 1 %} +{%- if hostname not in dmz_nodes %} +{%- set LOGSTASH.count = LOGSTASH.count + 1 %} - "{{ hostname }}:5644" #{{ node_details[hostname].ip }} +{%- endif %} {%- endfor %} {%- endif %} {%- if LOGSTASH.count > 1 %} diff --git a/salt/logstash/dmz_nodes.yaml b/salt/logstash/dmz_nodes.yaml new file mode 100644 index 000000000..982f72080 --- /dev/null +++ b/salt/logstash/dmz_nodes.yaml @@ -0,0 +1,9 @@ +# Do not edit this file. Copy it to /opt/so/saltstack/local/salt/logstash/ and make changes there. It should be formatted as a list. +# logstash: +# dmz_nodes: +# - mydmznodehostname1 +# - mydmznodehostname2 +# - mydmznodehostname3 + +logstash: + dmz_nodes: From e5b74bcb7890077f3a20ab96ac2e2de4ce2b98b5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 Jun 2022 15:26:25 -0400 Subject: [PATCH 3/3] remove podman state --- salt/podman/init.sls | 51 -------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 salt/podman/init.sls diff --git a/salt/podman/init.sls b/salt/podman/init.sls deleted file mode 100644 index f00a21074..000000000 --- a/salt/podman/init.sls +++ /dev/null @@ -1,51 +0,0 @@ -Pip pkg: - pkg.installed: - - name: python3-pip - -Podman pkg: - pkg.installed: - - name: podman - -Podman service: - file.managed: - - name: /usr/lib/systemd/system/podman.service - - source: salt://podman/podman.service - -Podman socket: - file.managed: - - name: /usr/lib/systemd/system/podman.socket - - source: salt://podman/podman.socket - service.running: - - name: podman.socket - - enable: true - -Docker socket: - file.symlink: - - name: /var/run/docker.sock - - target: /var/run/podman/podman.sock - -Docker python: - pip.installed: - - bin_env: /usr/bin/pip3 - - reload_modules: true - - pkgs: - - certifi==2019.11.28 - - chardet==3.0.4 - - docker==4.2.1 - - idna==2.9 - # - requests==2.23.0 - - six==1.14.0 - - urllib3==1.25.8 - - websocket-client==0.57.0 - -podman_docker_symlink: - file.symlink: - - name: /bin/docker - - target: /usr/bin/podman - -restart_salt_minion: - cmd.run: - - name: 'salt-call service.restart salt-minion' - - bg: true - - onchanges: - - pip: Docker python