From 502277b1b7456300eed950d85919558acd24f3d0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 23 May 2023 16:38:37 -0400 Subject: [PATCH 1/3] Hash check, use url base --- .../sbin_jinja/so-elastic-agent-gen-installers | 6 +++--- .../tools/sbin_jinja/so-elastic-fleet-setup | 11 +++++++++-- setup/so-functions | 14 +++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 2dd92d21b..b241eedb6 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -10,12 +10,12 @@ . /usr/sbin/so-common -FLEETHOST="https://{{ GLOBALS.manager_ip }}:8220" +#FLEETHOST="https://{{ GLOBALS.manager_ip }}:8220" for i in {1..30} do - ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints")) | .api_key') - #FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq -r '.items[].host_urls[]' | paste -sd ',') + ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') + FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq -r '.items[].host_urls[]' | paste -sd ',') if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi done if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL or Enrollment Token empty - exiting..." && exit; fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 6ad97a223..02c60165c 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -35,9 +35,16 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl printf "\n\n" {%- endif %} +# Add Manager IP & URL Base to Fleet Host URLs printf "\nAdd SO-Manager Fleet URL\n" +if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then + JSON_STRING=$( jq -n '{"fleet_server_hosts":["https://{{ GLOBALS.url_base }}:8220"]}') +else + JSON_STRING=$( jq -n '{"fleet_server_hosts":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.manager_ip }}:8220"]}') +fi + ## This array replaces whatever URLs are currently configured -curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220", "https://{{ GLOBALS.manager }}:8220"]}' +curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" @@ -74,7 +81,7 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl ### Finalization ### # Query for Enrollment Tokens for default policies -ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-default")) | .api_key') +ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key') # Store needed data in minion pillar diff --git a/setup/so-functions b/setup/so-functions index 09e219cfd..937c4e494 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -972,7 +972,19 @@ download_elastic_agent_artifacts() { else logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" - logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" + logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$version.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$version.md5" + + SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz | awk '{ print $1 }') + HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$version.md5) + + if [[ "$HASH" == "$SOURCEHASH" ]]; then + info "Elastic Agent source hash is good." + else + info "Unable to download the Elastic Agent source files." + exit 1 + fi + + logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" fi } From e1c361e555eea90e21e782928592074c72594a42 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 23 May 2023 16:50:40 -0400 Subject: [PATCH 2/3] Fix variable --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 937c4e494..86ff5f5b0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -972,10 +972,10 @@ download_elastic_agent_artifacts() { else logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" - logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$version.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$version.md5" + logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5" SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz | awk '{ print $1 }') - HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$version.md5) + HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.md5) if [[ "$HASH" == "$SOURCEHASH" ]]; then info "Elastic Agent source hash is good." From b3e0e688965db80e343113edd036e2ad89b4f53c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 24 May 2023 11:27:41 -0400 Subject: [PATCH 3/3] Change Fleet Host URL API --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index b241eedb6..84a519d37 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -15,7 +15,7 @@ for i in {1..30} do ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') - FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq -r '.items[].host_urls[]' | paste -sd ',') + FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',') if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi done if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL or Enrollment Token empty - exiting..." && exit; fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 02c60165c..c81d69282 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -38,13 +38,13 @@ printf "\n\n" # Add Manager IP & URL Base to Fleet Host URLs printf "\nAdd SO-Manager Fleet URL\n" if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then - JSON_STRING=$( jq -n '{"fleet_server_hosts":["https://{{ GLOBALS.url_base }}:8220"]}') + JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220"]}') else - JSON_STRING=$( jq -n '{"fleet_server_hosts":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.manager_ip }}:8220"]}') + JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.manager_ip }}:8220"]}') fi ## This array replaces whatever URLs are currently configured -curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/fleet_server_hosts" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n"