diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 2b312a536..c5dbca337 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -66,8 +66,8 @@ so-elastic-fleet: - FLEET_SERVER_SERVICE_TOKEN={{ SERVICETOKEN }} - FLEET_SERVER_POLICY_ID=FleetServer_{{ GLOBALS.hostname }} - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt - - FLEET_SERVER_CERT=/etc/pki/elasticfleet.crt - - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet.key + - FLEET_SERVER_CERT=/etc/pki/elasticfleet-server.crt + - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key - FLEET_CA=/etc/pki/tls/certs/intca.crt {% if DOCKER.containers['so-elastic-fleet'].extra_env %} {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index b1486e35c..c689e4e80 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -20,10 +20,10 @@ JSON_STRING=$( jq -n \ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -printf "\nCreate Logstash Output if node is not an Import or Eval install\n" +printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} -LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) -LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) +LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-agent.crt) +LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-agent.key) LOGSTASHCA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt) JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index ac937ca7b..6031ad529 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -59,8 +59,10 @@ so-logstash: - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro {% endif %} {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %} - - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro - - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro + - /etc/pki/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro + - /etc/pki/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro + - /etc/pki/elasticfleet-lumberjack.crt:/usr/share/logstash/elasticfleet-lumberjack.crt:ro + - /etc/pki/elasticfleet-lumberjack.p8:/usr/share/logstash/elasticfleet-lumberjack.key:ro {% endif %} {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %} - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro diff --git a/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf index f3257eb20..af42c86fb 100644 --- a/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf +++ b/salt/logstash/pipelines/config/so/0013_input_http_fleet.conf @@ -1,21 +1,18 @@ input { - http { - additional_codecs => { "application/json" => "json_lines" } + elastic_agent { port => 5056 tags => [ "elastic-agent" ] ssl => true - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - ssl_verify_mode => "peer" + ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt" + ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key" ecs_compatibility => v8 + id => "fleet-lumberjack-in" + codec => "json" + } +} +filter { + mutate { + rename => {"@metadata" => "metadata"} } } -filter { - if "elastic-agent" in [tags] { - mutate { - remove_field => ["http","[metadata][input]","url","user_agent"] -} - } -} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja b/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja index eec2bd74f..776488c06 100644 --- a/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja +++ b/salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja @@ -1,11 +1,15 @@ -output { - http { - url => 'https://{{ GLOBALS.manager }}:5056' - cacert => ["/usr/share/filebeat/ca.crt"] - http_method => post - retry_non_idempotent => true - format => json_batch - http_compression => true - ecs_compatibility => v8 +filter { + mutate { + add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}" + } } -} \ No newline at end of file + +output { + lumberjack { + codec => json + hosts => "{{ GLOBALS.manager }}" + ssl_certificate => "/usr/share/filebeat/ca.crt" + port => 5056 + id => "fleet-lumberjack-{{ GLOBALS.hostname }}" + } + } \ No newline at end of file diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4bb706d63..14efc8919 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -7,6 +7,7 @@ {% if sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} {% set global_ca_text = [] %} {% set global_ca_server = [] %} @@ -141,15 +142,16 @@ rediskeyperms: {% endif %} {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %} -# Create cert for Elastic Fleet Host +{% if grains['role'] not in [ 'so-heavynode', 'so-receiver'] %} +# Start -- Elastic Fleet Host Cert etc_elasticfleet_key: x509.private_key_managed: - - name: /etc/pki/elasticfleet.key + - name: /etc/pki/elasticfleet-server.key - keysize: 4096 - backup: True - new: True - {% if salt['file.file_exists']('/etc/pki/elasticfleet.key') -%} + {% if salt['file.file_exists']('/etc/pki/elasticfleet-server.key') -%} - prereq: - x509: etc_elasticfleet_crt {%- endif %} @@ -157,44 +159,33 @@ etc_elasticfleet_key: attempts: 5 interval: 30 -# Request a cert and drop it where it needs to go to be distributed etc_elasticfleet_crt: x509.certificate_managed: - - name: /etc/pki/elasticfleet.crt + - name: /etc/pki/elasticfleet-server.crt - ca_server: {{ ca_server }} - signing_policy: elasticfleet - - private_key: /etc/pki/elasticfleet.key + - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.hostname }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True -{% if grains.role not in ['so-heavynode'] %} - - unless: - # https://github.com/saltstack/salt/issues/52167 - # Will trigger 5 days (432000 sec) from cert expiration - - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticfleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' -{% endif %} - timeout: 30 - retry: attempts: 5 interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet.key -topk8 -out /etc/pki/elasticfleet.p8 -nocrypt" - - onchanges: - - x509: etc_elasticfleet_key efperms: file.managed: - replace: False - - name: /etc/pki/elasticfleet.key + - name: /etc/pki/elasticfleet-server.key - mode: 640 - group: 939 chownelasticfleetcrt: file.managed: - replace: False - - name: /etc/pki/elasticfleet.crt + - name: /etc/pki/elasticfleet-server.crt - mode: 640 - user: 947 - group: 939 @@ -202,32 +193,17 @@ chownelasticfleetcrt: chownelasticfleetkey: file.managed: - replace: False - - name: /etc/pki/elasticfleet.key + - name: /etc/pki/elasticfleet-server.key - mode: 640 - user: 947 - group: 939 +# End -- Elastic Fleet Host Cert -# Create Symlinks to the keys to distribute it to all the things -elasticfleetdircerts: - file.directory: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs - - makedirs: True - -efcrtlink: - file.symlink: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.crt - - target: /etc/pki/elasticfleet.crt - - user: socore - - group: socore - - -{% if grains.role not in ['so-fleet'] %} -# Create Cert for Elastic Fleet Logstash Input (Same cert used across all Fleet nodes) - -etc_elasticfleetlogstash_key: +# Start -- Elastic Fleet Logstash Input Cert +etc_elasticfleet_logstash_key: x509.private_key_managed: - name: /etc/pki/elasticfleet-logstash.key - - bits: 4096 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-logstash.key') -%} @@ -238,24 +214,17 @@ etc_elasticfleetlogstash_key: attempts: 5 interval: 30 -# Request a cert and drop it where it needs to go to be distributed -etc_elasticfleetlogstash_crt: +etc_elasticfleet_logstash_crt: x509.certificate_managed: - name: /etc/pki/elasticfleet-logstash.crt - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.hostname }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True -{% if grains.role not in ['so-heavynode'] %} - - unless: - # https://github.com/saltstack/salt/issues/52167 - # Will trigger 5 days (432000 sec) from cert expiration - - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' -{% endif %} - timeout: 30 - retry: attempts: 5 @@ -272,15 +241,7 @@ eflogstashperms: - mode: 640 - group: 939 -chownilogstashelasticfleetp8: - file.managed: - - replace: False - - name: /etc/pki/elasticfleet-logstash.p8 - - mode: 640 - - user: 947 - - group: 939 - -chownilogstashelasticfleetlogstashcrt: +chownelasticfleetlogstashcrt: file.managed: - replace: False - name: /etc/pki/elasticfleet-logstash.crt @@ -288,48 +249,143 @@ chownilogstashelasticfleetlogstashcrt: - user: 947 - group: 939 -chownilogstashelasticfleetlogstashkey: +chownelasticfleetlogstashkey: file.managed: - replace: False - name: /etc/pki/elasticfleet-logstash.key - mode: 640 - user: 947 - group: 939 +# End -- Elastic Fleet Logstash Input Cert +{% endif %} # endif is for not including HeavyNodes & Receivers -eflogstashkeylink: - file.symlink: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet-logstash.p8 - - target: /etc/pki/elasticfleet.p8 - - user: socore - - group: socore +# Start -- Elastic Fleet Node - Logstash Lumberjack Input / Output +# Cert needed on: Managers, Receivers +etc_elasticfleetlumberjack_key: + x509.private_key_managed: + - name: /etc/pki/elasticfleet-lumberjack.key + - bits: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/elasticfleet-lumberjack.key') -%} + - prereq: + - x509: etc_elasticfleet_crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 -eflogstashcrtlink: - file.symlink: - - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet-logstash.crt - - target: /etc/pki/elasticfleet.crt - - user: socore - - group: socore +etc_elasticfleetlumberjack_crt: + x509.certificate_managed: + - name: /etc/pki/elasticfleet-lumberjack.crt + - ca_server: {{ ca_server }} + - signing_policy: elasticfleet + - private_key: /etc/pki/elasticfleet-lumberjack.key + - CN: {{ GLOBALS.node_ip }} + - subjectAltName: DNS:{{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-lumberjack.key -topk8 -out /etc/pki/elasticfleet-lumberjack.p8 -nocrypt" + - onchanges: + - x509: etc_elasticfleet_key -{% endif %} - -/opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8: +eflogstashlumberjackperms: file.managed: - - replace: True - - source: salt://elasticfleet/files/certs/elasticfleet-logstash.p8 - - makedirs: True + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.key + - mode: 640 + - group: 939 + +chownilogstashelasticfleetlumberjackp8: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.p8 - mode: 640 - user: 931 - group: 939 -/opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt: +chownilogstashelasticfleetlogstashlumberjackcrt: file.managed: - - replace: True - - source: salt://elasticfleet/files/certs/elasticfleet-logstash.crt - - makedirs: True + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.crt - mode: 640 - user: 931 - group: 939 +chownilogstashelasticfleetlogstashlumberjackkey: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-lumberjack.key + - mode: 640 + - user: 931 + - group: 939 + +# End -- Elastic Fleet Node - Logstash Lumberjack Input / Output + +# Start -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output) +etc_elasticfleet_agent_key: + x509.private_key_managed: + - name: /etc/pki/elasticfleet-agent.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%} + - prereq: + - x509: etc_elasticfleet_crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +etc_elasticfleet_agent_crt: + x509.certificate_managed: + - name: /etc/pki/elasticfleet-agent.crt + - ca_server: {{ ca_server }} + - signing_policy: elasticfleet + - private_key: /etc/pki/elasticfleet-agent.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt" + - onchanges: + - x509: etc_elasticfleet_key + +efagentperms: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-agent.key + - mode: 640 + - group: 939 + +chownelasticfleetagentcrt: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-agent.crt + - mode: 640 + - user: 947 + - group: 939 + +chownelasticfleetagentkey: + file.managed: + - replace: False + - name: /etc/pki/elasticfleet-agent.key + - mode: 640 + - user: 947 + - group: 939 +# End -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output) + {% endif %} {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}