From 18a54b86f491d509877eea80911c29bcec50f4f2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 31 Jan 2023 14:57:39 -0500 Subject: [PATCH] More fixes --- pillar/logstash/fleet.sls | 2 +- pillar/logstash/init.sls | 1 + pillar/logstash/manager.sls | 6 +++--- pillar/top.sls | 4 ++++ salt/common/tools/sbin/so-minion | 5 +++-- salt/docker/defaults.yaml | 1 + salt/firewall/assigned_hostgroups.map.yaml | 3 +++ salt/firewall/ports/ports.yaml | 3 +++ salt/logstash/init.sls | 4 ++++ salt/logstash/map.jinja | 2 +- .../config/so/0013_input_lumberjack_fleet.conf | 13 +++++++++++++ .../so/9806_output_lumberjack_fleet.conf.jinja | 10 ++++++++++ 12 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf create mode 100644 salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja diff --git a/pillar/logstash/fleet.sls b/pillar/logstash/fleet.sls index 15641f935..fb70e7f0d 100644 --- a/pillar/logstash/fleet.sls +++ b/pillar/logstash/fleet.sls @@ -3,4 +3,4 @@ logstash: fleet: config: - so/0012_input_elastic_agent.conf - - so/9805_output_elastic_agent.conf.jinja \ No newline at end of file + - so/9806_output_lumberjack_fleet.conf.jinja \ No newline at end of file diff --git a/pillar/logstash/init.sls b/pillar/logstash/init.sls index 7ad31cf9b..b94ae2c44 100644 --- a/pillar/logstash/init.sls +++ b/pillar/logstash/init.sls @@ -4,6 +4,7 @@ logstash: - 0.0.0.0:3765:3765 - 0.0.0.0:5044:5044 - 0.0.0.0:5055:5055 + - 0.0.0.0:5056:5056 - 0.0.0.0:5644:5644 - 0.0.0.0:6050:6050 - 0.0.0.0:6051:6051 diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index 41a2197fd..cee8eec02 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -3,6 +3,6 @@ logstash: manager: config: - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/9999_output_redis.conf.jinja - + - so/0012_input_elastic_agent.conf + - so/0013_input_lumberjack_fleet.conf + - so/9999_output_redis.conf.jinja \ No newline at end of file diff --git a/pillar/top.sls b/pillar/top.sls index 96e0b7a53..1fb8c4e7d 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -204,6 +204,10 @@ base: - adv_global - backup.soc_backup - backup.adv_backup + - logstash + - logstash.fleet + - logstash.soc_logstash + - logstash.adv_logstash - minions.{{ grains.id }} - minions.adv_{{ grains.id }} diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index cce5e2a64..1c9aff028 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -131,8 +131,9 @@ function add_fleet_to_minion() { # Write out settings to minion file printf '%s\n'\ - "fleet-server:"\ - " ES-Token: '$ESTOKEN'"\ + "elasticfleet:"\ + " server:"\ + " es_token: '$ESTOKEN'"\ " " >> $PILLARFILE } diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index e2ec07d32..24a9d80a6 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -48,6 +48,7 @@ docker: - 0.0.0.0:3765:3765 - 0.0.0.0:5044:5044 - 0.0.0.0:5055:5055 + - 0.0.0.0:5056:5056 - 0.0.0.0:5644:5644 - 0.0.0.0:6050:6050 - 0.0.0.0:6051:6051 diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 82f183b3a..8d8d12035 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -249,6 +249,7 @@ role: - {{ portgroups.yum }} - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} + - {{ portgroups.beats_5056 }} - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.elastic_agent_control }} @@ -258,6 +259,7 @@ role: fleet: portgroups: - {{ portgroups.elasticsearch_rest }} + - {{ portgroups.beats_5056 }} sensors: portgroups: - {{ portgroups.docker_registry }} @@ -266,6 +268,7 @@ role: - {{ portgroups.yum }} - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} + - {{ portgroups.beats_5056 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} searchnodes: diff --git a/salt/firewall/ports/ports.yaml b/salt/firewall/ports/ports.yaml index d26b373cb..3f2407214 100644 --- a/salt/firewall/ports/ports.yaml +++ b/salt/firewall/ports/ports.yaml @@ -17,6 +17,9 @@ firewall: beats_5066: tcp: - 5066 + beats_5056: + tcp: + - 5056 docker_registry: tcp: - 5000 diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index fee8b5496..7720d3182 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -164,6 +164,10 @@ so-logstash: - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro {% endif %} + {% if GLOBALS.role in ['so-fleet'] %} + - /etc/pki/elasticfleet.crt:/usr/share/logstash/filebeat.crt:ro + - /etc/pki/elasticfleet02.p8:/usr/share/logstash/filebeat.key:ro + {% endif %} {% if GLOBALS.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %} - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro {% else %} diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 035e36d86..a70ab19d6 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,6 +1,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% set REDIS_NODES = [] %} -{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %} +{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch','so-fleet'] %} {% set node_data = salt['pillar.get']('logstash:nodes') %} {% for node_type, node_details in node_data.items() | sort %} {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} diff --git a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf new file mode 100644 index 000000000..894ecddb2 --- /dev/null +++ b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf @@ -0,0 +1,13 @@ +input { + http { + additional_codecs => { "application/json" => "json_lines" } + port => 5056 + tags => [ "elastic-agent" ] + ssl => true + ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] + ssl_certificate => "/usr/share/logstash/filebeat.crt" + ssl_key => "/usr/share/logstash/filebeat.key" + ssl_verify_mode => "peer" + ecs_compatibility => v8 + } +} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja new file mode 100644 index 000000000..c4bd28fe9 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja @@ -0,0 +1,10 @@ +output { + http { + url => 'https://{{ GLOBALS.manager }}:5056' + http_method => post + retry_non_idempotent => true + format => json_batch + http_compression => true + ecs_compatibility => v8 + } +} \ No newline at end of file