More fixes

pillar/logstash/fleet.sls

@@ -3,4 +3,4 @@ logstash:
     fleet:
       config:
         - so/0012_input_elastic_agent.conf     
-        - so/9805_output_elastic_agent.conf.jinja
+        - so/9806_output_lumberjack_fleet.conf.jinja

pillar/logstash/init.sls

@@ -4,6 +4,7 @@ logstash:
       - 0.0.0.0:3765:3765
       - 0.0.0.0:5044:5044
       - 0.0.0.0:5055:5055
+      - 0.0.0.0:5056:5056
       - 0.0.0.0:5644:5644
       - 0.0.0.0:6050:6050
       - 0.0.0.0:6051:6051

pillar/logstash/manager.sls

@@ -4,5 +4,5 @@ logstash:
       config:
         - so/0011_input_endgame.conf
         - so/0012_input_elastic_agent.conf
+        - so/0013_input_lumberjack_fleet.conf
         - so/9999_output_redis.conf.jinja
-        

pillar/top.sls

@@ -204,6 +204,10 @@ base:
     - adv_global
     - backup.soc_backup
     - backup.adv_backup
+    - logstash
+    - logstash.fleet
+    - logstash.soc_logstash
+    - logstash.adv_logstash
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 

salt/common/tools/sbin/so-minion

@@ -131,8 +131,9 @@ function add_fleet_to_minion() {
 
     # Write out settings to minion file
     printf '%s\n'\
-		"fleet-server:"\
+		"elasticfleet:"\
-		"  ES-Token: '$ESTOKEN'"\
+		"  server:"\
+		"    es_token: '$ESTOKEN'"\
 		"  " >> $PILLARFILE
 }
 

salt/docker/defaults.yaml

@@ -48,6 +48,7 @@ docker:
         - 0.0.0.0:3765:3765
         - 0.0.0.0:5044:5044
         - 0.0.0.0:5055:5055
+        - 0.0.0.0:5056:5056
         - 0.0.0.0:5644:5644
         - 0.0.0.0:6050:6050
         - 0.0.0.0:6051:6051

salt/firewall/assigned_hostgroups.map.yaml

@@ -249,6 +249,7 @@ role:
               - {{ portgroups.yum }}
               - {{ portgroups.beats_5044 }}
               - {{ portgroups.beats_5644 }}
+              - {{ portgroups.beats_5056 }}
               - {{ portgroups.redis }}
               - {{ portgroups.elasticsearch_node }}
               - {{ portgroups.elastic_agent_control }}
@@ -258,6 +259,7 @@ role:
           fleet:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+              - {{ portgroups.beats_5056 }}
           sensors:
             portgroups:
               - {{ portgroups.docker_registry }}
@@ -266,6 +268,7 @@ role:
               - {{ portgroups.yum }}
               - {{ portgroups.beats_5044 }}
               - {{ portgroups.beats_5644 }}
+              - {{ portgroups.beats_5056 }}
               - {{ portgroups.elastic_agent_control }}
               - {{ portgroups.elastic_agent_data }}
           searchnodes:

salt/firewall/ports/ports.yaml

@@ -17,6 +17,9 @@ firewall:
     beats_5066:
       tcp:
         - 5066
+    beats_5056:
+      tcp:
+        - 5056
     docker_registry:
       tcp:
         - 5000

salt/logstash/init.sls

@@ -164,6 +164,10 @@ so-logstash:
       - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
   {% endif %}
+  {% if GLOBALS.role in ['so-fleet'] %}
+      - /etc/pki/elasticfleet.crt:/usr/share/logstash/filebeat.crt:ro
+      - /etc/pki/elasticfleet02.p8:/usr/share/logstash/filebeat.key:ro
+  {% endif %}
   {% if GLOBALS.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
   {% else %}

salt/logstash/map.jinja

@@ -1,6 +1,6 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% set REDIS_NODES = [] %}
-{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %}
+{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch','so-fleet'] %}
   {% set node_data = salt['pillar.get']('logstash:nodes') %}
   {% for node_type, node_details in node_data.items() | sort %}
     {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}

salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf

@@ -0,0 +1,13 @@
+input {
+  http {
+    additional_codecs => { "application/json" => "json_lines" }
+    port => 5056
+    tags => [ "elastic-agent" ]
+    ssl => true
+    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
+    ssl_certificate => "/usr/share/logstash/filebeat.crt"
+    ssl_key => "/usr/share/logstash/filebeat.key"
+    ssl_verify_mode => "peer"
+    ecs_compatibility => v8
+  }
+}

salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja

@@ -0,0 +1,10 @@
+output {
+  http {
+    url => 'https://{{ GLOBALS.manager }}:5056'
+    http_method => post
+    retry_non_idempotent => true
+    format => json_batch
+    http_compression => true
+    ecs_compatibility => v8
+  }
+}