From 7561ec05127c796e4d48077a17ae4cd4865c9762 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 29 Jun 2023 08:52:51 -0400 Subject: [PATCH 1/9] Automatically manage Fleet Logstash Config --- salt/elasticfleet/defaults.yaml | 3 +- salt/elasticfleet/enabled.sls | 11 +++ salt/elasticfleet/soc_elasticfleet.yaml | 15 ++-- .../so-elastic-fleet-outputs-update | 75 +++++++++++++++++++ salt/manager/tools/sbin/so-minion | 2 +- 5 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 83261dc17..93b5eba9a 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,10 +2,11 @@ elasticfleet: enabled: False config: server: + custom_fqdn: '' + enable_auto_configuration: True endpoints_enrollment: '' es_token: '' grid_enrollment: '' - url: '' logging: zeek: excluded: diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index f388cb1c7..17dc8afa0 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -9,11 +9,22 @@ {% from 'docker/docker.map.jinja' import DOCKER %} {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} +{% set ENABLEAUTOCONFIGURATION = salt['pillar.get']('elasticfleet:config:server:enable_auto_configuration','') %} include: - elasticfleet.config - elasticfleet.sostatus +{% if ENABLEAUTOCONFIGURATION %} +so-elastic-fleet-auto-configure-logstash-outputs: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-outputs-update + +#so-elastic-fleet-auto-configure-server-urls: +# cmd.run: +# - name: /usr/sbin/so-elastic-fleet-urls-update +{% endif %} + {% if SERVICETOKEN != '' %} so-elastic-fleet: docker_container.running: diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 80b3a22b5..9b918f0ac 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -11,6 +11,16 @@ elasticfleet: helpLink: zeek.html config: server: + custom_fqdn: + description: Custom FQDN for Agents to connect to. + global: True + helpLink: elastic-fleet.html + advanced: True + enable_auto_configuration: + description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs. + global: True + helpLink: elastic-fleet.html + advanced: True endpoints_enrollment: description: Endpoint enrollment key. global: True @@ -29,8 +39,3 @@ elasticfleet: helpLink: elastic-fleet.html sensitive: True advanced: True - url: - description: Agent connection URL. - global: True - helpLink: elastic-fleet.html - advanced: True diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update new file mode 100644 index 000000000..b4df64d68 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -0,0 +1,75 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} + +function update_logstash_outputs() { + # Generate updated JSON payload + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') + + # Update Logstash Outputs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq +} + +# Get current list of Logstash Outputs +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash') + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON") +if [ "$CHECKSUM" != "so-manager_logstash" ]; then + printf "Failed to query for current Logstash Outputs..." + exit 1 +fi + +# Get the current list of Logstash outputs & hash them +CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") +CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + +# Create array & add initial elements +if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then + NEW_LIST=("{{ GLOBALS.url_base }}:5055") +else + NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.manager_ip }}:5055") +fi + +{% if CUSTOMFQDN != "" %} +# Add Custom Hostname to list +NEW_LIST+=("{{ CUSTOMFQDN }}:5055") +{% endif %} + +# Query for the current Grid Nodes that are running Logstash +LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') + +# Query for Receiver Nodes & add them to the list +if grep -q "receiver" <<< $LOGSTASHNODES; then + readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${RECEIVERNODES[@]}" + do + NEW_LIST+=("$NODE:5055") + done +fi + +# Query for Fleet Nodes & add them to the list +if grep -q "fleet" <<< $LOGSTASHNODES; then + readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${FLEETNODES[@]}" + do + NEW_LIST+=("$NODE:5055") + done +fi + +# Sort & hash the new list of Logstash Outputs +NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") +NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') + +# Compare the current & new list of outputs - if different, update the Logstash outputs +if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then + printf "\nHashes match - no update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + exit 0 +else + printf "\nHashes don't match - update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_logstash_outputs +fi diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ad2188644..edc0b1404 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -515,7 +515,7 @@ function createFLEET() { add_logstash_to_minion create_fleet_policy update_fleet_host_urls - update_logstash_outputs + #update_logstash_outputs add_telegraf_to_minion add_nginx_to_minion } From 4b069d91abd2044d2666658b825dde99cecbb73b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 29 Jun 2023 11:00:34 -0400 Subject: [PATCH 2/9] Check the correct pillar --- salt/elasticfleet/enabled.sls | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 17dc8afa0..2b312a536 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -7,15 +7,18 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} + +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{% set ELASTICFLEETMERGED = salt['pillar.get']('elasticfleet', ELASTICFLEETDEFAULTS.elasticfleet, merge=True) %} + {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} -{% set ENABLEAUTOCONFIGURATION = salt['pillar.get']('elasticfleet:config:server:enable_auto_configuration','') %} include: - elasticfleet.config - elasticfleet.sostatus -{% if ENABLEAUTOCONFIGURATION %} +{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %} so-elastic-fleet-auto-configure-logstash-outputs: cmd.run: - name: /usr/sbin/so-elastic-fleet-outputs-update From 1baea3bcd57bd182736532f404f6f2195c20f099 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 29 Jun 2023 17:24:52 -0400 Subject: [PATCH 3/9] Add Fleet to Logstash Nodes --- pillar/logstash/nodes.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index adf43156e..8d3bdab65 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -2,7 +2,7 @@ {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ', fun='network.ip_addrs', tgt_type='compound') | dictsort() %} From ff3bb11fbbbdae28a585425c2c9b3adc997644b7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 7 Jul 2023 16:44:16 -0400 Subject: [PATCH 4/9] Elastic Fleet Certs Refactor --- salt/elasticfleet/enabled.sls | 4 +- .../tools/sbin_jinja/so-elastic-fleet-setup | 6 +- salt/logstash/enabled.sls | 6 +- .../config/so/0013_input_http_fleet.conf | 23 +- .../so/9806_output_http_fleet.conf.jinja | 24 +- salt/ssl/init.sls | 216 +++++++++++------- 6 files changed, 169 insertions(+), 110 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 2b312a536..c5dbca337 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -66,8 +66,8 @@ so-elastic-fleet: - FLEET_SERVER_SERVICE_TOKEN={{ SERVICETOKEN }} - FLEET_SERVER_POLICY_ID=FleetServer_{{ GLOBALS.hostname }} - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt - - FLEET_SERVER_CERT=/etc/pki/elasticfleet.crt - - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet.key + - FLEET_SERVER_CERT=/etc/pki/elasticfleet-server.crt + - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key - FLEET_CA=/etc/pki/tls/certs/intca.crt {% if DOCKER.containers['so-elastic-fleet'].extra_env %} {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index b1486e35c..c689e4e80 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -20,10 +20,10 @@ JSON_STRING=$( jq -n \ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -printf "\nCreate Logstash Output if node is not an Import or Eval install\n" +printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} -LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) -LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) +LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-agent.crt) +LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-agent.key) LOGSTASHCA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt) JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index ac937ca7b..6031ad529 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -59,8 +59,10 @@ so-logstash: - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro {% endif %} {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %} - - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro - - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro + - /etc/pki/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro + - /etc/pki/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro + - /etc/pki/elasticfleet-lumberjack.crt:/usr/share/logstash/elasticfleet-lumberjack.crt:ro + - /etc/pki/elasticfleet-lumberjack.p8:/usr/share/logstash/elasticfleet-lumberjack.key:ro {% endif %} {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %} - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro diff --git a/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf index f3257eb20..af42c86fb 100644 --- a/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf +++ b/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf @@ -1,21 +1,18 @@ input { - http { - additional_codecs => { "application/json" => "json_lines" } + elastic_agent { port => 5056 tags => [ "elastic-agent" ] ssl => true - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - ssl_verify_mode => "peer" + ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt" + ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key" ecs_compatibility => v8 + id => "fleet-lumberjack-in" + codec => "json" + } +} +filter { + mutate { + rename => {"@metadata" => "metadata"} } } -filter { - if "elastic-agent" in [tags] { - mutate { - remove_field => ["http","[metadata][input]","url","user_agent"] -} - } -} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja b/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja index eec2bd74f..776488c06 100644 --- a/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja +++ b/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja @@ -1,11 +1,15 @@ -output { - http { - url => 'https://{{ GLOBALS.manager }}:5056' - cacert => ["/usr/share/filebeat/ca.crt"] - http_method => post - retry_non_idempotent => true - format => json_batch - http_compression => true - ecs_compatibility => v8 +filter { + mutate { + add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}" + } } -} \ No newline at end of file + +output { + lumberjack { + codec => json + hosts => "{{ GLOBALS.manager }}" + ssl_certificate => "/usr/share/filebeat/ca.crt" + port => 5056 + id => "fleet-lumberjack-{{ GLOBALS.hostname }}" + } + } \ No newline at end of file diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4bb706d63..14efc8919 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -7,6 +7,7 @@ {% if sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} {% set global_ca_text = [] %} {% set global_ca_server = [] %} @@ -141,15 +142,16 @@ rediskeyperms: {% endif %} {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %} -# Create cert for Elastic Fleet Host +{% if grains['role'] not in [ 'so-heavynode', 'so-receiver'] %} +# Start -- Elastic Fleet Host Cert etc_elasticfleet_key: x509.private_key_managed: - - name: /etc/pki/elasticfleet.key + - name: /etc/pki/elasticfleet-server.key - keysize: 4096 - backup: True - new: True - {% if salt['file.file_exists']('/etc/pki/elasticfleet.key') -%} + {% if salt['file.file_exists']('/etc/pki/elasticfleet-server.key') -%} - prereq: - x509: etc_elasticfleet_crt {%- endif %} @@ -157,44 +159,33 @@ etc_elasticfleet_key: attempts: 5 interval: 30 -# Request a cert and drop it where it needs to go to be distributed etc_elasticfleet_crt: x509.certificate_managed: - - name: /etc/pki/elasticfleet.crt + - name: /etc/pki/elasticfleet-server.crt - ca_server: {{ ca_server }} - signing_policy: elasticfleet - - private_key: /etc/pki/elasticfleet.key + - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.hostname }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True -{% if grains.role not in ['so-heavynode'] %} - - unless: - # https://github.com/saltstack/salt/issues/52167 - # Will trigger 5 days (432000 sec) from cert expiration - - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticfleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' -{% endif %} - timeout: 30 - retry: attempts: 5 interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet.key -topk8 -out /etc/pki/elasticfleet.p8 -nocrypt" - - onchanges: - - x509: etc_elasticfleet_key efperms: file.managed: - replace: False - - name: /etc/pki/elasticfleet.key + - name: /etc/pki/elasticfleet-server.key - mode: 640 - group: 939 chownelasticfleetcrt: file.managed: - replace: False - - name: /etc/pki/elasticfleet.crt + - name: /etc/pki/elasticfleet-server.crt - mode: 640 - user: 947 - group: 939 @@ -202,32 +193,17 @@ chownelasticfleetcrt: chownelasticfleetkey: file.managed: - replace: False - - name: /etc/pki/elasticfleet.key + - name: /etc/pki/elasticfleet-server.key - mode: 640 - user: 947 - group: 939 +# End -- Elastic Fleet Host Cert -# Create Symlinks to the keys to distribute it to all the things -elasticfleetdircerts: - file.directory: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs - - makedirs: True - -efcrtlink: - file.symlink: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.crt - - target: /etc/pki/elasticfleet.crt - - user: socore - - group: socore - - -{% if grains.role not in ['so-fleet'] %} -# Create Cert for Elastic Fleet Logstash Input (Same cert used across all Fleet nodes) - -etc_elasticfleetlogstash_key: +# Start -- Elastic Fleet Logstash Input Cert +etc_elasticfleet_logstash_key: x509.private_key_managed: - name: /etc/pki/elasticfleet-logstash.key - - bits: 4096 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-logstash.key') -%} @@ -238,24 +214,17 @@ etc_elasticfleetlogstash_key: attempts: 5 interval: 30 -# Request a cert and drop it where it needs to go to be distributed -etc_elasticfleetlogstash_crt: +etc_elasticfleet_logstash_crt: x509.certificate_managed: - name: /etc/pki/elasticfleet-logstash.crt - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.hostname }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True -{% if grains.role not in ['so-heavynode'] %} - - unless: - # https://github.com/saltstack/salt/issues/52167 - # Will trigger 5 days (432000 sec) from cert expiration - - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' -{% endif %} - timeout: 30 - retry: attempts: 5 @@ -272,15 +241,7 @@ eflogstashperms: - mode: 640 - group: 939 -chownilogstashelasticfleetp8: - file.managed: - - replace: False - - name: /etc/pki/elasticfleet-logstash.p8 - - mode: 640 - - user: 947 - - group: 939 - -chownilogstashelasticfleetlogstashcrt: +chownelasticfleetlogstashcrt: file.managed: - replace: False - name: /etc/pki/elasticfleet-logstash.crt @@ -288,48 +249,143 @@ chownilogstashelasticfleetlogstashcrt: - user: 947 - group: 939 -chownilogstashelasticfleetlogstashkey: +chownelasticfleetlogstashkey: file.managed: - replace: False - name: /etc/pki/elasticfleet-logstash.key - mode: 640 - user: 947 - group: 939 +# End -- Elastic Fleet Logstash Input Cert +{% endif %} # endif is for not including HeavyNodes & Receivers -eflogstashkeylink: - file.symlink: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet-logstash.p8 - - target: /etc/pki/elasticfleet.p8 - - user: socore - - group: socore +# Start -- Elastic Fleet Node - Logstash Lumberjack Input / Output +# Cert needed on: Managers, Receivers +etc_elasticfleetlumberjack_key: + x509.private_key_managed: + - name: /etc/pki/elasticfleet-lumberjack.key + - bits: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/elasticfleet-lumberjack.key') -%} + - prereq: + - x509: etc_elasticfleet_crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 -eflogstashcrtlink: - file.symlink: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet-logstash.crt - - target: /etc/pki/elasticfleet.crt - - user: socore - - group: socore +etc_elasticfleetlumberjack_crt: + x509.certificate_managed: + - name: /etc/pki/elasticfleet-lumberjack.crt + - ca_server: {{ ca_server }} + - signing_policy: elasticfleet + - private_key: /etc/pki/elasticfleet-lumberjack.key + - CN: {{ GLOBALS.node_ip }} + - subjectAltName: DNS:{{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-lumberjack.key -topk8 -out /etc/pki/elasticfleet-lumberjack.p8 -nocrypt" + - onchanges: + - x509: etc_elasticfleet_key -{% endif %} - -/opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8: +eflogstashlumberjackperms: file.managed: - - replace: True - - source: salt://elasticfleet/files/certs/elasticfleet-logstash.p8 - - makedirs: True + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.key + - mode: 640 + - group: 939 + +chownilogstashelasticfleetlumberjackp8: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.p8 - mode: 640 - user: 931 - group: 939 -/opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt: +chownilogstashelasticfleetlogstashlumberjackcrt: file.managed: - - replace: True - - source: salt://elasticfleet/files/certs/elasticfleet-logstash.crt - - makedirs: True + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.crt - mode: 640 - user: 931 - group: 939 +chownilogstashelasticfleetlogstashlumberjackkey: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.key + - mode: 640 + - user: 931 + - group: 939 + +# End -- Elastic Fleet Node - Logstash Lumberjack Input / Output + +# Start -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output) +etc_elasticfleet_agent_key: + x509.private_key_managed: + - name: /etc/pki/elasticfleet-agent.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%} + - prereq: + - x509: etc_elasticfleet_crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +etc_elasticfleet_agent_crt: + x509.certificate_managed: + - name: /etc/pki/elasticfleet-agent.crt + - ca_server: {{ ca_server }} + - signing_policy: elasticfleet + - private_key: /etc/pki/elasticfleet-agent.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt" + - onchanges: + - x509: etc_elasticfleet_key + +efagentperms: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-agent.key + - mode: 640 + - group: 939 + +chownelasticfleetagentcrt: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-agent.crt + - mode: 640 + - user: 947 + - group: 939 + +chownelasticfleetagentkey: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-agent.key + - mode: 640 + - user: 947 + - group: 939 +# End -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output) + {% endif %} {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %} From e8860a7d2cab1d84a5d79d88c9130bb25b23dc1d Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 8 Jul 2023 09:04:55 -0400 Subject: [PATCH 5/9] Fix perms --- salt/logstash/enabled.sls | 2 +- salt/ssl/init.sls | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 6031ad529..595297510 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -60,7 +60,7 @@ so-logstash: {% endif %} {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %} - /etc/pki/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro - - /etc/pki/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro + - /etc/pki/elasticfleet-logstash.key:/usr/share/logstash/elasticfleet-logstash.key:ro - /etc/pki/elasticfleet-lumberjack.crt:/usr/share/logstash/elasticfleet-lumberjack.crt:ro - /etc/pki/elasticfleet-lumberjack.p8:/usr/share/logstash/elasticfleet-lumberjack.key:ro {% endif %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 9b83892cb..496c05b42 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -234,7 +234,7 @@ chownelasticfleetlogstashcrt: - replace: False - name: /etc/pki/elasticfleet-logstash.crt - mode: 640 - - user: 947 + - user: 931 - group: 939 chownelasticfleetlogstashkey: @@ -242,7 +242,7 @@ chownelasticfleetlogstashkey: - replace: False - name: /etc/pki/elasticfleet-logstash.key - mode: 640 - - user: 947 + - user: 931 - group: 939 # End -- Elastic Fleet Logstash Input Cert {% endif %} # endif is for not including HeavyNodes & Receivers From ce1f363424af599284cbccfe60c7894d55165653 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 8 Jul 2023 13:30:19 -0400 Subject: [PATCH 6/9] Allow base_url --- salt/elasticsearch/config.map.jinja | 4 ++-- salt/ssl/init.sls | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 1b8728bf5..ed4a5033f 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -3,11 +3,11 @@ {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} -{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes are removed #} +{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #} {% set ES_LOGSTASH_NODES = [] %} {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} {% for node_type, node_details in node_data.items() | sort %} -{% if node_type != 'heavynode' %} +{% if node_type not in ['heavynode', 'fleet'] %} {% for hostname in node_data[node_type].keys() %} {% do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} {% endfor %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 496c05b42..f51c51039 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -153,8 +153,8 @@ etc_elasticfleet_crt: - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - - CN: {{ GLOBALS.hostname }} - - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - CN: {{ GLOBALS.url_base }} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -208,8 +208,8 @@ etc_elasticfleet_logstash_crt: - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - - CN: {{ GLOBALS.hostname }} - - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - CN: {{ GLOBALS.url_base }} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 8c16feb77266f9d4830b9c3dc6cf6afdf0033ba6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 9 Jul 2023 12:22:55 -0400 Subject: [PATCH 7/9] Rename Fleet pipelines --- salt/logstash/defaults.yaml | 5 +++-- salt/logstash/enabled.sls | 2 +- ...put_http_fleet.conf => 0013_input_lumberjack_fleet.conf} | 0 ...t.conf.jinja => 9806_output_lumberjack_fleet.conf.jinja} | 0 salt/ssl/init.sls | 6 ++++-- 5 files changed, 8 insertions(+), 5 deletions(-) rename salt/logstash/pipelines/config/so/{0013_input_http_fleet.conf => 0013_input_lumberjack_fleet.conf} (100%) rename salt/logstash/pipelines/config/so/{9806_output_http_fleet.conf.jinja => 9806_output_lumberjack_fleet.conf.jinja} (100%) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 8d27730b2..574a4f826 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -22,15 +22,16 @@ logstash: defined_pipelines: fleet: - so/0012_input_elastic_agent.conf - - so/9806_output_http_fleet.conf.jinja + - so/9806_output_lumberjack_fleet.conf.jinja manager: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - - so/0013_input_http_fleet.conf + - so/0013_input_lumberjack_fleet.conf - so/9999_output_redis.conf.jinja receiver: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf + - so/0013_input_lumberjack_fleet.conf - so/9999_output_redis.conf.jinja search: - so/0900_input_redis.conf.jinja diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 595297510..a88e97b19 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -62,7 +62,7 @@ so-logstash: - /etc/pki/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro - /etc/pki/elasticfleet-logstash.key:/usr/share/logstash/elasticfleet-logstash.key:ro - /etc/pki/elasticfleet-lumberjack.crt:/usr/share/logstash/elasticfleet-lumberjack.crt:ro - - /etc/pki/elasticfleet-lumberjack.p8:/usr/share/logstash/elasticfleet-lumberjack.key:ro + - /etc/pki/elasticfleet-lumberjack.key:/usr/share/logstash/elasticfleet-lumberjack.key:ro {% endif %} {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %} - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro diff --git a/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf similarity index 100% rename from salt/logstash/pipelines/config/so/0013_input_http_fleet.conf rename to salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf diff --git a/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja rename to salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index f51c51039..97e971b83 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -186,7 +186,9 @@ chownelasticfleetkey: - user: 947 - group: 939 # End -- Elastic Fleet Host Cert +{% endif %} # endif is for not including HeavyNodes & Receivers +{% if grains['role'] not in [ 'so-heavynode'] %} # Start -- Elastic Fleet Logstash Input Cert etc_elasticfleet_logstash_key: x509.private_key_managed: @@ -220,7 +222,7 @@ etc_elasticfleet_logstash_crt: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-logstash.key -topk8 -out /etc/pki/elasticfleet-logstash.p8 -nocrypt" - onchanges: - - x509: etc_elasticfleet_key + - x509: etc_elasticfleet_logstash_key eflogstashperms: file.managed: @@ -245,7 +247,7 @@ chownelasticfleetlogstashkey: - user: 931 - group: 939 # End -- Elastic Fleet Logstash Input Cert -{% endif %} # endif is for not including HeavyNodes & Receivers +{% endif %} # endif is for not including HeavyNodes # Start -- Elastic Fleet Node - Logstash Lumberjack Input / Output # Cert needed on: Managers, Receivers From 7805ca8beb85565742175e9c19c422ded66f7fdd Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 10 Jul 2023 10:38:14 -0400 Subject: [PATCH 8/9] Add Failover Support --- salt/firewall/defaults.yaml | 4 +++- .../so/9806_output_lumberjack_fleet.conf.jinja | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 45567de91..0a4635004 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -731,7 +731,6 @@ firewall: - yum - beats_5044 - beats_5644 - - beats_5056 - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -1159,6 +1158,9 @@ firewall: chain: DOCKER-USER: hostgroups: + fleet: + portgroups: + - beats_5056 sensor: portgroups: - beats_5044 diff --git a/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja index 776488c06..50328e833 100644 --- a/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja +++ b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja @@ -1,3 +1,13 @@ +{% set FAILOVER_LOGSTASH_NODES = [] %} +{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% for node_type, node_details in node_data.items() | sort %} +{% if node_type not in ['heavynode', 'fleet', 'searchnode'] %} +{% for hostname in node_data[node_type].keys() %} +{% do FAILOVER_LOGSTASH_NODES.append(node_details[hostname].ip) %} +{% endfor %} +{% endif %} +{% endfor %} + filter { mutate { add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}" @@ -7,7 +17,7 @@ filter { output { lumberjack { codec => json - hosts => "{{ GLOBALS.manager }}" + hosts => {{ FAILOVER_LOGSTASH_NODES }} ssl_certificate => "/usr/share/filebeat/ca.crt" port => 5056 id => "fleet-lumberjack-{{ GLOBALS.hostname }}" From 31edf2e8ea3c31f144500e6a39e3a7930be9ca79 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 10 Jul 2023 14:17:42 -0400 Subject: [PATCH 9/9] Tighten & Document Pipelines --- salt/logstash/defaults.yaml | 14 +++++++------- .../config/so/0013_input_lumberjack_fleet.conf | 6 +++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 574a4f826..da141b7d2 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -21,21 +21,21 @@ logstash: - fleet defined_pipelines: fleet: - - so/0012_input_elastic_agent.conf - - so/9806_output_lumberjack_fleet.conf.jinja + - so/0012_input_elastic_agent.conf # Logs from agents + - so/9806_output_lumberjack_fleet.conf.jinja # Logstash to Logstash Output manager: - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/0013_input_lumberjack_fleet.conf + - so/0012_input_elastic_agent.conf # Logs from agents + - so/0013_input_lumberjack_fleet.conf # Logstash to Logstash Input - so/9999_output_redis.conf.jinja receiver: - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/0013_input_lumberjack_fleet.conf + - so/0012_input_elastic_agent.conf # Logs from agents + - so/0013_input_lumberjack_fleet.conf # Logstash to Logstash Input - so/9999_output_redis.conf.jinja search: - so/0900_input_redis.conf.jinja - - so/9805_output_elastic_agent.conf.jinja + - so/9805_output_elastic_agent.conf.jinja # Elastic Agent data Output to ES (Final) - so/9900_output_endgame.conf.jinja custom0: [] custom1: [] diff --git a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf index af42c86fb..0377a81c4 100644 --- a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf +++ b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf @@ -1,7 +1,7 @@ input { elastic_agent { port => 5056 - tags => [ "elastic-agent" ] + tags => [ "elastic-agent", "fleet-lumberjack-input" ] ssl => true ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt" ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key" @@ -10,9 +10,13 @@ input { codec => "json" } } + + filter { + if "fleet-lumberjack-input" in [tags] { mutate { rename => {"@metadata" => "metadata"} } } +}