From 0d4f6b4fe6517006d1cb302f057adb99ccee0f57 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 13 Jun 2023 16:32:19 -0400 Subject: [PATCH 1/3] Change Elastic Fleet Tarball naming --- .../sbin_jinja/so-elastic-agent-gen-installers | 13 ++++++++++--- setup/so-functions | 12 ++++++------ setup/so-variables | 3 +++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 4910ceda1..703784fce 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -12,11 +12,18 @@ for i in {1..30} do + ELASTICVERSION=$(so-elasticsearch-query / | jq -r .version.number) ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',') -if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi +if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]] && [[ $ELASTICVERSION ]]; then break; else sleep 10; fi done -if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL or Enrollment Token empty - exiting..." && exit; fi + +if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]] || [[ -z $ELASTICVERSION ]] +then + printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." + printf "\nFleet Host: $FLEETHOST, Elastic Version: $ELASTICVERSION, Enrollment Token: $ENROLLMENTOKEN\n" + exit +fi OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" ) @@ -25,7 +32,7 @@ rm -rf /nsm/elastic-agent-workspace mkdir -p /nsm/elastic-agent-workspace printf "\n### Extracting outer tarball and then each individual tarball/zip\n" -tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-{{ GLOBALS.so_version }}.tar.gz -C /nsm/elastic-agent-workspace/ +tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTICVERSION.tar.gz -C /nsm/elastic-agent-workspace/ unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/ for archive in /nsm/elastic-agent-workspace/*.tar.gz do diff --git a/setup/so-functions b/setup/so-functions index a9d5b434e..a71bb223f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -976,14 +976,14 @@ detect_os() { download_elastic_agent_artifacts() { if [[ $is_iso ]]; then - logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" + logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" else logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" "" "" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$elastic_agent_tarball_version.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5" "" "" - SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz | awk '{ print $1 }') - HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5) + SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz | awk '{ print $1 }') + HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$elastic_agent_tarball_version.md5) if [[ "$HASH" == "$SOURCEHASH" ]]; then info "Elastic Agent source hash is good." @@ -992,7 +992,7 @@ download_elastic_agent_artifacts() { fail_setup fi - logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" + logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" fi } diff --git a/setup/so-variables b/setup/so-variables index 2c7cb3dba..82bef510f 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -219,3 +219,6 @@ export patch_pillar_file adv_patch_pillar_file="$local_salt_dir/pillar/patch/adv_patch.sls" export adv_patch_pillar_file + +elastic_agent_tarball_version="8.7.1" +export elastic_agent_tarball_version \ No newline at end of file From 9947f9def4e5d220616a48cf0ee6db1e86b096fd Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 14 Jun 2023 07:38:03 -0400 Subject: [PATCH 2/3] Rework tarball naming schema --- salt/common/tools/sbin/so-common | 1 + .../tools/sbin_jinja/so-elastic-agent-gen-installers | 9 ++++----- setup/so-functions | 12 ++++++------ setup/so-variables | 5 +---- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 523a1b230..f25bdb431 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -5,6 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +ELASTIC_AGENT_TARBALL_VERSION="8.7.1" DEFAULT_SALT_DIR=/opt/so/saltstack/default DOC_BASE_URL="https://docs.securityonion.net/en/2.4" diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 703784fce..704f1537a 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -8,20 +8,19 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} +. /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common for i in {1..30} do - ELASTICVERSION=$(so-elasticsearch-query / | jq -r .version.number) ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',') if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]] && [[ $ELASTICVERSION ]]; then break; else sleep 10; fi done -if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]] || [[ -z $ELASTICVERSION ]] -then +if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." - printf "\nFleet Host: $FLEETHOST, Elastic Version: $ELASTICVERSION, Enrollment Token: $ENROLLMENTOKEN\n" + printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n" exit fi @@ -32,7 +31,7 @@ rm -rf /nsm/elastic-agent-workspace mkdir -p /nsm/elastic-agent-workspace printf "\n### Extracting outer tarball and then each individual tarball/zip\n" -tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTICVERSION.tar.gz -C /nsm/elastic-agent-workspace/ +tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-agent-workspace/ unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/ for archive in /nsm/elastic-agent-workspace/*.tar.gz do diff --git a/setup/so-functions b/setup/so-functions index a71bb223f..ff012b151 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -976,14 +976,14 @@ detect_os() { download_elastic_agent_artifacts() { if [[ $is_iso ]]; then - logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" + logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" else logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" "" "" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$elastic_agent_tarball_version.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5" "" "" - SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz | awk '{ print $1 }') - HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$elastic_agent_tarball_version.md5) + SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz | awk '{ print $1 }') + HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5) if [[ "$HASH" == "$SOURCEHASH" ]]; then info "Elastic Agent source hash is good." @@ -992,7 +992,7 @@ download_elastic_agent_artifacts() { fail_setup fi - logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$elastic_agent_tarball_version.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" + logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" fi } diff --git a/setup/so-variables b/setup/so-variables index 82bef510f..7c5e51c6c 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -218,7 +218,4 @@ patch_pillar_file="$local_salt_dir/pillar/patch/soc_patch.sls" export patch_pillar_file adv_patch_pillar_file="$local_salt_dir/pillar/patch/adv_patch.sls" -export adv_patch_pillar_file - -elastic_agent_tarball_version="8.7.1" -export elastic_agent_tarball_version \ No newline at end of file +export adv_patch_pillar_file \ No newline at end of file From fedfbe9fec9299eed18ecda722c127ebc876d282 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 14 Jun 2023 08:52:56 -0400 Subject: [PATCH 3/3] Fix tarball output name --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ff012b151..1b15bb140 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -979,8 +979,8 @@ download_elastic_agent_artifacts() { logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" else logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" "" "" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" "" "" SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz | awk '{ print $1 }') HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5)