Elastic Fleet Certs Refactor

salt/elasticfleet/enabled.sls

@@ -66,8 +66,8 @@ so-elastic-fleet:
       - FLEET_SERVER_SERVICE_TOKEN={{ SERVICETOKEN }}
       - FLEET_SERVER_POLICY_ID=FleetServer_{{ GLOBALS.hostname }}
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      - FLEET_SERVER_CERT=/etc/pki/elasticfleet.crt
+      - FLEET_SERVER_CERT=/etc/pki/elasticfleet-server.crt
-      - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet.key
+      - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt
       {% if DOCKER.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %}

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -20,10 +20,10 @@ JSON_STRING=$( jq -n \
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 printf "\n\n"
 
-printf "\nCreate Logstash Output if node is not an Import or Eval install\n"
+printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n"
 {% if grains.role not in ['so-import', 'so-eval'] %}
-LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
+LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-agent.crt)
-LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
+LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-agent.key)
 LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
 JSON_STRING=$( jq -n \
                   --arg LOGSTASHCRT "$LOGSTASHCRT" \

salt/logstash/enabled.sls

@@ -59,8 +59,10 @@ so-logstash:
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
       {% endif %}
       {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %}
-      - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro
+      - /etc/pki/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro
-      - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro
+      - /etc/pki/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro
+      - /etc/pki/elasticfleet-lumberjack.crt:/usr/share/logstash/elasticfleet-lumberjack.crt:ro
+      - /etc/pki/elasticfleet-lumberjack.p8:/usr/share/logstash/elasticfleet-lumberjack.key:ro
       {% endif %}
       {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro

salt/logstash/pipelines/config/so/0013_input_http_fleet.conf

@@ -1,21 +1,18 @@
 input {
-  http {
+  elastic_agent {
-    additional_codecs => { "application/json" => "json_lines" }
     port => 5056
     tags => [ "elastic-agent" ]
     ssl => true
-    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
+    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
-    ssl_certificate => "/usr/share/logstash/filebeat.crt"
+    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
-    ssl_key => "/usr/share/logstash/filebeat.key"
-    ssl_verify_mode => "peer"
     ecs_compatibility => v8
+    id => "fleet-lumberjack-in"  
+    codec => "json"
+  }
+}
+filter {
+  mutate {
+    rename => {"@metadata" => "metadata"}
   }
 }
 
-filter {
-   if "elastic-agent" in [tags] {
-  mutate {
-    remove_field => ["http","[metadata][input]","url","user_agent"]
-}
-   }
-}

salt/logstash/pipelines/config/so/9806_output_http_fleet.conf.jinja

@@ -1,11 +1,15 @@
-output {
+filter {
-  http {
+       mutate {
-    url => 'https://{{ GLOBALS.manager }}:5056'
+                    add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}"
-    cacert => ["/usr/share/filebeat/ca.crt"]
+          }
-    http_method => post
-    retry_non_idempotent => true
-    format => json_batch
-    http_compression => true
-    ecs_compatibility => v8
   }
-}
+
+output { 
+    lumberjack { 
+        codec => json 
+        hosts => "{{ GLOBALS.manager }}"
+        ssl_certificate => "/usr/share/filebeat/ca.crt"
+        port => 5056 
+        id => "fleet-lumberjack-{{ GLOBALS.hostname }}"
+        } 
+    }

salt/ssl/init.sls

@@ -7,6 +7,7 @@
 {% if sls in allowed_states %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
+{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %}
 
 {% set global_ca_text = [] %}
 {% set global_ca_server = [] %}
@@ -141,15 +142,16 @@ rediskeyperms:
 {% endif %}
 
 {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %}
-# Create cert for Elastic Fleet Host
 
+{% if grains['role'] not in [ 'so-heavynode', 'so-receiver'] %}
+# Start -- Elastic Fleet Host Cert
 etc_elasticfleet_key:
   x509.private_key_managed:
-    - name: /etc/pki/elasticfleet.key
+    - name: /etc/pki/elasticfleet-server.key
     - keysize: 4096
     - backup: True
     - new: True
-    {% if salt['file.file_exists']('/etc/pki/elasticfleet.key') -%}
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-server.key') -%}
     - prereq:
       - x509: etc_elasticfleet_crt
     {%- endif %}
@@ -157,44 +159,33 @@ etc_elasticfleet_key:
         attempts: 5
         interval: 30
 
-# Request a cert and drop it where it needs to go to be distributed
 etc_elasticfleet_crt:
   x509.certificate_managed:
-    - name: /etc/pki/elasticfleet.crt
+    - name: /etc/pki/elasticfleet-server.crt
     - ca_server: {{ ca_server }}
     - signing_policy: elasticfleet
-    - private_key: /etc/pki/elasticfleet.key
+    - private_key: /etc/pki/elasticfleet-server.key
     - CN: {{ GLOBALS.hostname }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
-{% if grains.role not in ['so-heavynode'] %}
-    - unless:
-      # https://github.com/saltstack/salt/issues/52167
-      # Will trigger 5 days (432000 sec) from cert expiration
-      - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticfleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
-{% endif %}
     - timeout: 30
     - retry:
         attempts: 5
         interval: 30
-  cmd.run:
-    - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet.key -topk8 -out /etc/pki/elasticfleet.p8 -nocrypt"
-    - onchanges:
-      - x509: etc_elasticfleet_key
 
 efperms:
   file.managed:
     - replace: False
-    - name: /etc/pki/elasticfleet.key
+    - name: /etc/pki/elasticfleet-server.key
     - mode: 640
     - group: 939
 
 chownelasticfleetcrt:
   file.managed:
     - replace: False
-    - name: /etc/pki/elasticfleet.crt
+    - name: /etc/pki/elasticfleet-server.crt
     - mode: 640
     - user: 947
     - group: 939
@@ -202,32 +193,17 @@ chownelasticfleetcrt:
 chownelasticfleetkey:
   file.managed:
     - replace: False
-    - name: /etc/pki/elasticfleet.key
+    - name: /etc/pki/elasticfleet-server.key
     - mode: 640
     - user: 947
     - group: 939
+# End -- Elastic Fleet Host Cert
 
-# Create Symlinks to the keys to distribute it to all the things
+# Start -- Elastic Fleet Logstash Input Cert
-elasticfleetdircerts:
+etc_elasticfleet_logstash_key:
-  file.directory:
-    - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs
-    - makedirs: True
-
-efcrtlink:
-  file.symlink:
-    - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.crt
-    - target: /etc/pki/elasticfleet.crt
-    - user: socore
-    - group: socore
-
-
-{% if grains.role not in ['so-fleet'] %}
-# Create Cert for Elastic Fleet Logstash Input (Same cert used across all Fleet nodes)
-
-etc_elasticfleetlogstash_key:
   x509.private_key_managed:
     - name: /etc/pki/elasticfleet-logstash.key
-    - bits: 4096
+    - keysize: 4096
     - backup: True
     - new: True
     {% if salt['file.file_exists']('/etc/pki/elasticfleet-logstash.key') -%}
@@ -238,24 +214,17 @@ etc_elasticfleetlogstash_key:
         attempts: 5
         interval: 30
 
-# Request a cert and drop it where it needs to go to be distributed
+etc_elasticfleet_logstash_crt:
-etc_elasticfleetlogstash_crt:
   x509.certificate_managed:
     - name: /etc/pki/elasticfleet-logstash.crt
     - ca_server: {{ ca_server }}
     - signing_policy: elasticfleet
     - private_key: /etc/pki/elasticfleet-logstash.key
     - CN: {{ GLOBALS.hostname }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - subjectAltName: IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
-{% if grains.role not in ['so-heavynode'] %}
-    - unless:
-      # https://github.com/saltstack/salt/issues/52167
-      # Will trigger 5 days (432000 sec) from cert expiration
-      - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
-{% endif %}
     - timeout: 30
     - retry:
         attempts: 5
@@ -272,15 +241,7 @@ eflogstashperms:
     - mode: 640
     - group: 939
 
-chownilogstashelasticfleetp8:
+chownelasticfleetlogstashcrt:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/elasticfleet-logstash.p8
-    - mode: 640
-    - user: 947
-    - group: 939
-
-chownilogstashelasticfleetlogstashcrt:
   file.managed:
     - replace: False
     - name: /etc/pki/elasticfleet-logstash.crt
@@ -288,48 +249,143 @@ chownilogstashelasticfleetlogstashcrt:
     - user: 947
     - group: 939
 
-chownilogstashelasticfleetlogstashkey:
+chownelasticfleetlogstashkey:
   file.managed:
     - replace: False
     - name: /etc/pki/elasticfleet-logstash.key
     - mode: 640
     - user: 947
     - group: 939
+# End -- Elastic Fleet Logstash Input Cert
+{% endif %} # endif is for not including HeavyNodes & Receivers 
 
-eflogstashkeylink:
+# Start -- Elastic Fleet Node - Logstash Lumberjack Input / Output
-  file.symlink:
+# Cert needed on: Managers, Receivers
-    - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet-logstash.p8
+etc_elasticfleetlumberjack_key:
-    - target: /etc/pki/elasticfleet.p8
+  x509.private_key_managed:
-    - user: socore
+    - name: /etc/pki/elasticfleet-lumberjack.key
-    - group: socore
+    - bits: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-lumberjack.key') -%}
+    - prereq:
+      - x509: etc_elasticfleet_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
 
-eflogstashcrtlink:
+etc_elasticfleetlumberjack_crt:
-  file.symlink:
+  x509.certificate_managed:
-    - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet-logstash.crt
+    - name: /etc/pki/elasticfleet-lumberjack.crt
-    - target: /etc/pki/elasticfleet.crt
+    - ca_server: {{ ca_server }}
-    - user: socore
+    - signing_policy: elasticfleet
-    - group: socore
+    - private_key: /etc/pki/elasticfleet-lumberjack.key
+    - CN: {{ GLOBALS.node_ip }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-lumberjack.key -topk8 -out /etc/pki/elasticfleet-lumberjack.p8 -nocrypt"
+    - onchanges:
+      - x509: etc_elasticfleet_key
 
-{% endif %}
+eflogstashlumberjackperms:
-
-/opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:
   file.managed:
-    - replace: True
+    - replace: False
-    - source: salt://elasticfleet/files/certs/elasticfleet-logstash.p8
+    - name: /etc/pki/elasticfleet-lumberjack.key
-    - makedirs: True
+    - mode: 640
+    - group: 939
+
+chownilogstashelasticfleetlumberjackp8:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-lumberjack.p8
     - mode: 640
     - user: 931
     - group: 939
 
-/opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:
+chownilogstashelasticfleetlogstashlumberjackcrt:
   file.managed:
-    - replace: True
+    - replace: False
-    - source: salt://elasticfleet/files/certs/elasticfleet-logstash.crt
+    - name: /etc/pki/elasticfleet-lumberjack.crt
-    - makedirs: True
     - mode: 640
     - user: 931
     - group: 939
 
+chownilogstashelasticfleetlogstashlumberjackkey:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-lumberjack.key
+    - mode: 640
+    - user: 931
+    - group: 939
+
+# End -- Elastic Fleet Node - Logstash Lumberjack Input / Output
+
+# Start -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
+etc_elasticfleet_agent_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-agent.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%}
+    - prereq:
+      - x509: etc_elasticfleet_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+etc_elasticfleet_agent_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-agent.crt
+    - ca_server: {{ ca_server }}
+    - signing_policy: elasticfleet
+    - private_key: /etc/pki/elasticfleet-agent.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt"
+    - onchanges:
+      - x509: etc_elasticfleet_key
+
+efagentperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.key
+    - mode: 640
+    - group: 939
+
+chownelasticfleetagentcrt:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.crt
+    - mode: 640
+    - user: 947
+    - group: 939
+
+chownelasticfleetagentkey:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.key
+    - mode: 640
+    - user: 947
+    - group: 939
+# End -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
+
 {% endif %}
 
 {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}