From 40294e276215d7546ac37bf57f003b0f392803cd Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 12 Jul 2023 08:49:36 -0400 Subject: [PATCH 01/15] Update Integration JSON --- .../endpoints-initial/system-endpoints.json | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json index a5890794a..b6a21249a 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json @@ -13,9 +13,14 @@ "system.auth": { "enabled": true, "vars": { + "ignore_older": "72h", "paths": [ "/var/log/auth.log*", "/var/log/secure*" + ], + "preserve_original_event": false, + "tags": [ + "system-auth" ] } }, @@ -24,34 +29,49 @@ "vars": { "paths": [ "/var/log/messages*", - "/var/log/syslog*" - ] + "/var/log/syslog*", + "/var/log/system*" + ], + "tags": [], + "ignore_older": "72h" } } } }, "system-winlog": { "enabled": true, - "vars": { - "preserve_original_event": false - }, "streams": { "system.application": { "enabled": true, "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, "tags": [] } }, "system.security": { "enabled": true, "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, + "tags": [] + } + }, + "system.system": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, "tags": [] } } - } - }, - "system-system/metrics": { - "enabled": false + } + }, + "system-system/metrics": { + "enabled": false } } } From 43d63a3187dd29969a8f361d3695254777bf124e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Jul 2023 10:59:12 -0400 Subject: [PATCH 02/15] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 35cee72dc..79a614418 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.3 +2.4.4 From 85bb5a327cca4de8c63aee936d967e2ea7482fdc Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 13 Jul 2023 09:38:09 -0400 Subject: [PATCH 03/15] Fix long vs float for pe version --- salt/elasticsearch/files/ingest/strelka.file | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index 04b6c7119..eb6568123 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -64,6 +64,7 @@ { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 70 && ctx.rule?.score <=89", "field": "event.severity", "value": 3, "override": true } }, { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 90", "field": "event.severity", "value": 4, "override": true } }, { "set": { "if": "ctx.scan?.entropy?.entropy == 0", "field": "scan.entropy.entropy", "value": 0.0, "override": true } }, + { "set": { "if": "ctx.scan?.pe?.image_version == 0", "field": "scan.pe.image_version", "value": 0.0, "override": true } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" }}, { "convert" : { "field" : "scan.exiftool","type": "string", "ignore_missing":true }}, { "remove": { "field": ["host", "path", "message", "exiftool", "scan.yara.meta"], "ignore_missing": true } }, From e3249c8e4ca206c713094ca01523dad0fbb055f2 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 13 Jul 2023 14:18:57 +0000 Subject: [PATCH 04/15] Wrap values in quotes for proper conversion --- salt/elasticsearch/files/ingest/strelka.file | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index eb6568123..741e20aa1 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -63,8 +63,8 @@ { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 50 && ctx.rule?.score <=69", "field": "event.severity", "value": 2, "override": true } }, { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 70 && ctx.rule?.score <=89", "field": "event.severity", "value": 3, "override": true } }, { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 90", "field": "event.severity", "value": 4, "override": true } }, - { "set": { "if": "ctx.scan?.entropy?.entropy == 0", "field": "scan.entropy.entropy", "value": 0.0, "override": true } }, - { "set": { "if": "ctx.scan?.pe?.image_version == 0", "field": "scan.pe.image_version", "value": 0.0, "override": true } }, + { "set": { "if": "ctx.scan?.entropy?.entropy == 0", "field": "scan.entropy.entropy", "value": "0.0", "override": true } }, + { "set": { "if": "ctx.scan?.pe?.image_version == 0", "field": "scan.pe.image_version", "value": "0.0", "override": true } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" }}, { "convert" : { "field" : "scan.exiftool","type": "string", "ignore_missing":true }}, { "remove": { "field": ["host", "path", "message", "exiftool", "scan.yara.meta"], "ignore_missing": true } }, From 41f00c0aa1c1f657161ec7f0e8f42d73fd286946 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 13 Jul 2023 14:32:22 +0000 Subject: [PATCH 05/15] Add agent scripts --- .../tools/sbin_jinja/so-elastic-agent-inspect | 16 ++++++++++++++++ .../tools/sbin_jinja/so-elastic-agent-restart | 16 ++++++++++++++++ .../tools/sbin_jinja/so-elastic-agent-start | 17 +++++++++++++++++ .../tools/sbin_jinja/so-elastic-agent-status | 17 +++++++++++++++++ .../tools/sbin_jinja/so-elastic-agent-stop | 17 +++++++++++++++++ .../tools/sbin_jinja/so-elastic-agent-version | 17 +++++++++++++++++ 6 files changed, 100 insertions(+) create mode 100755 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect create mode 100755 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-restart create mode 100755 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-start create mode 100755 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status create mode 100755 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-stop create mode 100755 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect new file mode 100755 index 000000000..370b70a69 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-elastic-fleet-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent /bin/elastic-agent inspect +{% else %} +/bin/elastic-agent inspect +{% endif %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-restart b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-restart new file mode 100755 index 000000000..717df6fbd --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-restart @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-elastic-fleet-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent service elastic-agent restart +{% else %} +service elastic-agent restart +{% endif %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-start b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-start new file mode 100755 index 000000000..bd4829332 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-start @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-elastic-fleet-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent service elastic-agent start +{% else %} +service elastic-agent start +{% endif %} + diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status new file mode 100755 index 000000000..f27c63b9f --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-elastic-fleet-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent /bin/elastic-agent status +{% else %} +/bin/elastic-agent status +{% endif %} + diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-stop b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-stop new file mode 100755 index 000000000..c974d3e06 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-stop @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-elastic-fleet-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent service elastic-agent stop +{% else %} +service elastic-agent stop +{% endif %} + diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version new file mode 100755 index 000000000..781250a30 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-elastic-fleet-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent /bin/elastic-agent version +{% else %} +/bin/elastic-agent version +{% endif %} + From 0c177ec9239c1dc0bd2d6a7651f559d1310d9f28 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 13 Jul 2023 10:46:59 -0400 Subject: [PATCH 06/15] Allow Elasticsearch integration policy --- .../tools/sbin/so-elastic-fleet-integration-policy-load | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 54538ed9e..49bfb69ac 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -35,9 +35,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" else printf "\n\nIntegration does not exist - Creating integration\n" - if [ "$NAME" != "elasticsearch-logs" ]; then - elastic_fleet_integration_create "@$INTEGRATION" - fi + elastic_fleet_integration_create "@$INTEGRATION" fi done if [[ "$RETURN_CODE" != "1" ]]; then From fb8a79e112d300f16f44054acbccc7c048a0c940 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 13 Jul 2023 13:15:01 -0400 Subject: [PATCH 07/15] Retry install --- salt/elasticfleet/install_agent_grid.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 1d6c5a241..8b5bb4c3e 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -14,12 +14,14 @@ run_installer: - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 - cwd: /opt/so - args: -token={{ GRIDNODETOKENGENERAL }} + - retry: True {% else %} run_installer: cmd.script: - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 - cwd: /opt/so - args: -token={{ GRIDNODETOKENHEAVY }} + - retry: True {% endif %} {% endif %} From 78db64a41986ab269bc8ce47e923f573b65b26f5 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 14 Jul 2023 08:40:26 -0400 Subject: [PATCH 08/15] Auto-managed Fleet Server URLs --- salt/elasticfleet/enabled.sls | 12 ++- .../so-elastic-fleet-outputs-update | 12 ++- .../tools/sbin_jinja/so-elastic-fleet-setup | 6 +- .../sbin_jinja/so-elastic-fleet-urls-update | 75 +++++++++++++++++++ salt/logstash/defaults.yaml | 6 +- ...nf => 0012_input_elastic_agent.conf.jinja} | 2 +- 6 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update rename salt/logstash/pipelines/config/so/{0012_input_elastic_agent.conf => 0012_input_elastic_agent.conf.jinja} (86%) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 4eeae5ec3..c2e1ca494 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -16,14 +16,18 @@ include: - elasticfleet.config - elasticfleet.sostatus -{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %} +# If enabled, automatically update Fleet Logstash Outputs +{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} so-elastic-fleet-auto-configure-logstash-outputs: cmd.run: - name: /usr/sbin/so-elastic-fleet-outputs-update +{% endif %} -#so-elastic-fleet-auto-configure-server-urls: -# cmd.run: -# - name: /usr/sbin/so-elastic-fleet-urls-update +# If enabled, automatically update Fleet Server URLs & ES Connection +{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %} +so-elastic-fleet-auto-configure-server-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-urls-update {% endif %} {% if SERVICETOKEN != '' %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index b4df64d68..042084d84 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -4,6 +4,14 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + function update_logstash_outputs() { # Generate updated JSON payload JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') @@ -27,10 +35,10 @@ CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') # Create array & add initial elements -if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then +if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then NEW_LIST=("{{ GLOBALS.url_base }}:5055") else - NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.manager_ip }}:5055") + NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055") fi {% if CUSTOMFQDN != "" %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index c689e4e80..763b97bee 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -35,12 +35,12 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl printf "\n\n" {%- endif %} -# Add Manager IP & URL Base to Fleet Host URLs +# Add Manager Hostname & URL Base to Fleet Host URLs printf "\nAdd SO-Manager Fleet URL\n" -if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then +if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220"]}') else - JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.manager_ip }}:8220"]}') + JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.hostname }}:8220"]}') fi ## This array replaces whatever URLs are currently configured diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update new file mode 100644 index 000000000..343b32442 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -0,0 +1,75 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +function update_fleet_urls() { + # Generate updated JSON payload + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-default","is_default":true,"host_urls": $UPDATEDLIST}') + + # Update Fleet Server URLs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/fleet_server_hosts/grid-default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} + +# Get current list of Fleet Server URLs +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default') + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON") +if [ "$CHECKSUM" != "grid-default" ]; then + printf "Failed to query for current Fleet Server URLs..." + exit 1 +fi + +# Get the current list of Fleet Server URLs & hash them +CURRENT_LIST=$(jq -c -r '.item.host_urls' <<< "$RAW_JSON") +CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + +# IP or Hostname?? +# Create array & add initial elements +if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then + NEW_LIST=("https://{{ GLOBALS.url_base }}:8220") +else + NEW_LIST=("https://{{ GLOBALS.url_base }}:8220" "https://{{ GLOBALS.hostname }}:8220") +fi + +{% if CUSTOMFQDN != "" %} +# Add Custom Hostname to list +NEW_LIST+=("https://{{ CUSTOMFQDN }}:8220") +{% endif %} + +# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) +LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') + +# Query for Fleet Nodes & add them to the list (Hostname) +if grep -q "fleet" <<< $LOGSTASHNODES; then + readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${FLEETNODES[@]}" + do + NEW_LIST+=("https://$NODE:8220") + done +fi + +# Sort & hash the new list of Fleet Server URLs +NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") +NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') + +# Compare the current & new list of URLs - if different, update the Fleet Server URLs +if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then + printf "\nHashes match - no update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + exit 0 +else + printf "\nHashes don't match - update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_fleet_urls +fi diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 5a75cf099..e4c18cc64 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -21,16 +21,16 @@ logstash: - fleet defined_pipelines: fleet: - - so/0012_input_elastic_agent.conf + - so/0012_input_elastic_agent.conf.jinja - so/9806_output_lumberjack_fleet.conf.jinja manager: - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf + - so/0012_input_elastic_agent.conf.jinja - so/0013_input_lumberjack_fleet.conf - so/9999_output_redis.conf.jinja receiver: - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf + - so/0012_input_elastic_agent.conf.jinja - so/0013_input_lumberjack_fleet.conf - so/9999_output_redis.conf.jinja search: diff --git a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja similarity index 86% rename from salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf rename to salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja index d8c420bcd..035436a96 100644 --- a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf +++ b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja @@ -1,7 +1,7 @@ input { elastic_agent { port => 5055 - tags => [ "elastic-agent" ] + tags => [ "elastic-agent", "input-{{ GLOBALS.hostname }}" ] ssl => true ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] ssl_certificate => "/usr/share/logstash/elasticfleet-logstash.crt" From 2cc0c4c0ac9ad472ffcbd0518cc04a0bfaab5c55 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 14 Jul 2023 12:07:32 -0400 Subject: [PATCH 09/15] Automatically Update ES URLs --- salt/elasticfleet/enabled.sls | 7 +++ .../sbin_jinja/so-elastic-fleet-es-url-update | 53 +++++++++++++++++++ .../sbin_jinja/so-elastic-fleet-urls-update | 1 - 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index c2e1ca494..aff9b25d2 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -30,6 +30,13 @@ so-elastic-fleet-auto-configure-server-urls: - name: /usr/sbin/so-elastic-fleet-urls-update {% endif %} +# Automatically update Fleet Server Elasticsearch URLs +{% if grains.role not in ['so-fleet'] %} +so-elastic-fleet-auto-configure-elasticsearch-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-es-url-update +{% endif %} + {% if SERVICETOKEN != '' %} so-elastic-fleet: docker_container.running: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update new file mode 100644 index 000000000..6acda746c --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -0,0 +1,53 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +function update_es_urls() { + # Generate updated JSON payload + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":false,"is_default_monitoring":false,"config_yaml":""}') + + # Update Fleet Elasticsearch URLs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} + +# Get current list of Fleet Elasticsearch URLs +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch') + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON") +if [ "$CHECKSUM" != "so-manager_elasticsearch" ]; then + printf "Failed to query for current Fleet Server Elasticsearch URLs..." + exit 1 +fi + +# Get the current list of Fleet Server Elasticsearch & hash them +CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") +CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + +# Create array & add initial elements +NEW_LIST=("https://{{ GLOBALS.hostname }}:9200") + + +# Sort & hash the new list of Fleet Elasticsearch URLs +NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") +NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') + +# Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs +if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then + printf "\nHashes match - no update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + exit 0 +else + printf "\nHashes don't match - update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_es_urls +fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 343b32442..24c5dabed 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -34,7 +34,6 @@ fi CURRENT_LIST=$(jq -c -r '.item.host_urls' <<< "$RAW_JSON") CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') -# IP or Hostname?? # Create array & add initial elements if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then NEW_LIST=("https://{{ GLOBALS.url_base }}:8220") From 8cf3ceeb715d2dd9a6df2f1881d2b45461514772 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 14 Jul 2023 21:43:03 +0000 Subject: [PATCH 10/15] Update agent scripts --- salt/elasticagent/config.sls | 18 ++++- .../so-elastic-agent-gen-installers | 70 +++++++++++++++++++ .../tools/sbin_jinja/so-elastic-agent-inspect | 16 +++++ .../so-elastic-agent-restart | 6 ++ .../so-elastic-agent-start | 5 ++ .../tools/sbin_jinja/so-elastic-agent-status | 17 +++++ .../so-elastic-agent-stop | 5 ++ .../tools/sbin_jinja/so-elastic-agent-version | 17 +++++ .../tools/sbin_jinja/so-elastic-agent-inspect | 2 +- .../tools/sbin_jinja/so-elastic-agent-status | 2 +- .../tools/sbin_jinja/so-elastic-agent-version | 2 +- 11 files changed, 156 insertions(+), 4 deletions(-) create mode 100755 salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers create mode 100755 salt/elasticagent/tools/sbin_jinja/so-elastic-agent-inspect rename salt/elasticagent/tools/{sbin => sbin_jinja}/so-elastic-agent-restart (80%) rename salt/elasticagent/tools/{sbin => sbin_jinja}/so-elastic-agent-start (80%) create mode 100755 salt/elasticagent/tools/sbin_jinja/so-elastic-agent-status rename salt/elasticagent/tools/{sbin => sbin_jinja}/so-elastic-agent-stop (80%) create mode 100755 salt/elasticagent/tools/sbin_jinja/so-elastic-agent-version diff --git a/salt/elasticagent/config.sls b/salt/elasticagent/config.sls index 18d0e482e..d681f1c81 100644 --- a/salt/elasticagent/config.sls +++ b/salt/elasticagent/config.sls @@ -28,6 +28,23 @@ elasticagentconfdir: - group: 939 - makedirs: True +elasticagent_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://elasticagent/tools/sbin + - user: 949 + - group: 939 + - file_mode: 755 + +elasticagent_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://elasticagent/tools/sbin_jinja + - user: 949 + - group: 939 + - file_mode: 755 + - template: jinja + # Create config create-elastic-agent-config: file.managed: @@ -37,7 +54,6 @@ create-elastic-agent-config: - group: 939 - template: jinja - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers new file mode 100755 index 000000000..2a19dcbd9 --- /dev/null +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -0,0 +1,70 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. + +#so-elastic-agent-gen-installers $FleetHostURLs $EnrollmentToken + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common +. /usr/sbin/so-elastic-fleet-common + +for i in {1..30} +do + ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') + FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',') +if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi +done + +if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then + printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." + printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n" + exit +fi + +OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" ) + +printf "\n### Creating a temp directory at /nsm/elastic-agent-workspace\n" +rm -rf /nsm/elastic-agent-workspace +mkdir -p /nsm/elastic-agent-workspace + +printf "\n### Extracting outer tarball and then each individual tarball/zip\n" +tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-agent-workspace/ +unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/ +for archive in /nsm/elastic-agent-workspace/*.tar.gz +do + printf "\nExtracting $archive..." + tar xf "$archive" -C /nsm/elastic-agent-workspace/ +done + +printf "\n### Stripping out unused components" +find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete + +printf "\n### Tarring everything up again" +for OS in "${OSARCH[@]}" +do + printf "\nCreating tarball for $OS..." + rm -rf /nsm/elastic-agent-workspace/elastic-agent + mv /nsm/elastic-agent-workspace/elastic-agent-*-$OS /nsm/elastic-agent-workspace/elastic-agent + tar -czf /nsm/elastic-agent-workspace/$OS.tar.gz -C /nsm/elastic-agent-workspace elastic-agent +done + +GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" ) +GOARCH="amd64" +printf "\n### Generating OS packages using the cleaned up tarballs" +for GOOS in "${GOTARGETOS[@]}" +do + if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi + printf "\n\n### Generating $GOOS/$GOARCH Installer...\n" + docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \ + --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \ + --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ + --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ + {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH} + printf "\n### $GOOS/$GOARCH Installer Generated...\n" +done + +printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace" +rm -rf /nsm/elastic-agent-workspace diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-inspect b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-inspect new file mode 100755 index 000000000..27b4cbbeb --- /dev/null +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-inspect @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent /usr/share/elastic-agent/elastic-agent inspect +{% else %} +/bin/elastic-agent inspect +{% endif %} diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-restart b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart similarity index 80% rename from salt/elasticagent/tools/sbin/so-elastic-agent-restart rename to salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart index cb125ce10..00d5421ef 100755 --- a/salt/elasticagent/tools/sbin/so-elastic-agent-restart +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart @@ -5,6 +5,12 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. + + . /usr/sbin/so-common +{% if grains.role == 'so-heavynode' %} /usr/sbin/so-restart elastic-agent $1 +{% else %} +service elastic-agent restart +{% endif %} diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-start b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start similarity index 80% rename from salt/elasticagent/tools/sbin/so-elastic-agent-start rename to salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start index 294b9ba7a..fc9c50cb6 100755 --- a/salt/elasticagent/tools/sbin/so-elastic-agent-start +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start @@ -9,4 +9,9 @@ . /usr/sbin/so-common +{% if grains.role == 'so-heavynode' %} /usr/sbin/so-start elastic-agent $1 +{% else %} +service elastic-agent start +{% endif %} + diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-status b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-status new file mode 100755 index 000000000..2c617c8a8 --- /dev/null +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-status @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent /usr/share/elastic-agent/elastic-agent status +{% else %} +/bin/elastic-agent status +{% endif %} + diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-stop b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-stop similarity index 80% rename from salt/elasticagent/tools/sbin/so-elastic-agent-stop rename to salt/elasticagent/tools/sbin_jinja/so-elastic-agent-stop index cbfdcd380..863cb5194 100755 --- a/salt/elasticagent/tools/sbin/so-elastic-agent-stop +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-stop @@ -9,4 +9,9 @@ . /usr/sbin/so-common +{% if grains.role == 'so-heavynode' %} /usr/sbin/so-stop elastic-agent $1 +{% else %} +service elastic-agent stop +{% endif %} + diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-version b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-version new file mode 100755 index 000000000..6e2cc537e --- /dev/null +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-version @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +{% if grains.role == 'so-heavynode' %} +docker exec so-elastic-agent /usr/share/elastic-agent/elastic-agent version +{% else %} +/bin/elastic-agent version +{% endif %} + diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect index 370b70a69..2ad0a3379 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-inspect @@ -10,7 +10,7 @@ . /usr/sbin/so-elastic-fleet-common {% if grains.role == 'so-heavynode' %} -docker exec so-elastic-agent /bin/elastic-agent inspect +docker exec so-elastic-agent /usr/share/elastic-agent/elastic-agent inspect {% else %} /bin/elastic-agent inspect {% endif %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status index f27c63b9f..52a155ab2 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-status @@ -10,7 +10,7 @@ . /usr/sbin/so-elastic-fleet-common {% if grains.role == 'so-heavynode' %} -docker exec so-elastic-agent /bin/elastic-agent status +docker exec so-elastic-agent /usr/share/elastic-agent/elastic-agent status {% else %} /bin/elastic-agent status {% endif %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version index 781250a30..1b152048e 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-version @@ -10,7 +10,7 @@ . /usr/sbin/so-elastic-fleet-common {% if grains.role == 'so-heavynode' %} -docker exec so-elastic-agent /bin/elastic-agent version +docker exec so-elastic-agent /usr/share/elastic-agent/elastic-agent version {% else %} /bin/elastic-agent version {% endif %} From c723a091073ae3c236548b5f6e4e2d9de420e252 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 14 Jul 2023 21:45:25 +0000 Subject: [PATCH 11/15] Remove agent installer generation script --- .../so-elastic-agent-gen-installers | 70 ------------------- 1 file changed, 70 deletions(-) delete mode 100755 salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers deleted file mode 100755 index 2a19dcbd9..000000000 --- a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-gen-installers +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use -# this file except in compliance with the Elastic License 2.0. - -#so-elastic-agent-gen-installers $FleetHostURLs $EnrollmentToken - -{% from 'vars/globals.map.jinja' import GLOBALS %} - -. /usr/sbin/so-common -. /usr/sbin/so-elastic-fleet-common - -for i in {1..30} -do - ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') - FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',') -if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi -done - -if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then - printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." - printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n" - exit -fi - -OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" ) - -printf "\n### Creating a temp directory at /nsm/elastic-agent-workspace\n" -rm -rf /nsm/elastic-agent-workspace -mkdir -p /nsm/elastic-agent-workspace - -printf "\n### Extracting outer tarball and then each individual tarball/zip\n" -tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-agent-workspace/ -unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/ -for archive in /nsm/elastic-agent-workspace/*.tar.gz -do - printf "\nExtracting $archive..." - tar xf "$archive" -C /nsm/elastic-agent-workspace/ -done - -printf "\n### Stripping out unused components" -find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete - -printf "\n### Tarring everything up again" -for OS in "${OSARCH[@]}" -do - printf "\nCreating tarball for $OS..." - rm -rf /nsm/elastic-agent-workspace/elastic-agent - mv /nsm/elastic-agent-workspace/elastic-agent-*-$OS /nsm/elastic-agent-workspace/elastic-agent - tar -czf /nsm/elastic-agent-workspace/$OS.tar.gz -C /nsm/elastic-agent-workspace elastic-agent -done - -GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" ) -GOARCH="amd64" -printf "\n### Generating OS packages using the cleaned up tarballs" -for GOOS in "${GOTARGETOS[@]}" -do - if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi - printf "\n\n### Generating $GOOS/$GOARCH Installer...\n" - docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \ - --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \ - --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ - --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ - {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH} - printf "\n### $GOOS/$GOARCH Installer Generated...\n" -done - -printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace" -rm -rf /nsm/elastic-agent-workspace From e715dfa3548b37b13b69b8dd96aff2d436cb1917 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 17 Jul 2023 14:27:39 +0000 Subject: [PATCH 12/15] Remove sbin --- salt/elasticagent/config.sls | 8 -------- 1 file changed, 8 deletions(-) diff --git a/salt/elasticagent/config.sls b/salt/elasticagent/config.sls index d681f1c81..8b24f3b22 100644 --- a/salt/elasticagent/config.sls +++ b/salt/elasticagent/config.sls @@ -28,14 +28,6 @@ elasticagentconfdir: - group: 939 - makedirs: True -elasticagent_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://elasticagent/tools/sbin - - user: 949 - - group: 939 - - file_mode: 755 - elasticagent_sbin_jinja: file.recurse: - name: /usr/sbin From 31696803e1d9becdfda503de21656d1236cc8713 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 17 Jul 2023 14:28:12 +0000 Subject: [PATCH 13/15] Use correct name --- salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start index fc9c50cb6..47ee83691 100755 --- a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-start @@ -10,7 +10,7 @@ . /usr/sbin/so-common {% if grains.role == 'so-heavynode' %} -/usr/sbin/so-start elastic-agent $1 +/usr/sbin/so-start elasticagent $1 {% else %} service elastic-agent start {% endif %} From 1a4515fc8a1b4fdbcaccce50590bbca75388b73f Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 17 Jul 2023 14:30:51 +0000 Subject: [PATCH 14/15] Split restart into stop and start --- salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart index 00d5421ef..825f3b0fa 100755 --- a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart @@ -10,7 +10,8 @@ . /usr/sbin/so-common {% if grains.role == 'so-heavynode' %} -/usr/sbin/so-restart elastic-agent $1 +/usr/sbin/so-stop elastic-agent $1 +/usr/sbin/so-start elastic-agent $1 {% else %} service elastic-agent restart {% endif %} From d19615f74387b6aded127cea9995b0e7f601c9c5 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 17 Jul 2023 14:42:27 +0000 Subject: [PATCH 15/15] Fix typo --- salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart index 825f3b0fa..ea6724e69 100755 --- a/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart +++ b/salt/elasticagent/tools/sbin_jinja/so-elastic-agent-restart @@ -11,7 +11,7 @@ {% if grains.role == 'so-heavynode' %} /usr/sbin/so-stop elastic-agent $1 -/usr/sbin/so-start elastic-agent $1 +/usr/sbin/so-start elasticagent $1 {% else %} service elastic-agent restart {% endif %}