diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 431aa6f97..7ed727b48 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -10,6 +10,15 @@ {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %} +so-elastic-agent-install: + file.managed: + - name: /usr/sbin/so-elastic-agent-install + - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install + - user: 947 + - group: 939 + - mode: 755 + - show_changes: False + {% if not AGENT_STATUS or not AGENT_EXISTS %} pull_agent_installer: diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index c6a499e94..8f57d217c 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -9,14 +9,17 @@ # passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later GRIDNODETOKEN="$1" +LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log" check_agent_health() { - timeout=180 + timeout=300 interval=10 start=$SECONDS while (( SECONDS - start < timeout )); do - if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then + agent_status=$(elastic-agent status 2>&1) + echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE" + if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then return 0 fi echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..." @@ -65,6 +68,8 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 + truncate -s 0 "$LOGFILE" + uninstall_agent while [[ $attempts -lt 3 ]]; do diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index d25c18e29..de342657f 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -30,7 +30,7 @@ done if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n" - exit + exit 1 fi OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" ) @@ -62,31 +62,54 @@ do done GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" ) -GOARCH="amd64" printf "\n### Generating OS packages using the cleaned up tarballs" -for GOOS in "${GOTARGETOS[@]}" -do +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi printf "\n\n### Generating $GOOS/$GOARCH Installer...\n" docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \ --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \ --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ - --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ + --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH} printf "\n### $GOOS/$GOARCH Installer Generated...\n" done printf "\n\n### Generating MSI...\n" -cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe docker run \ ---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \ +--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" +# Verify installers were created +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" + if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi + if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then + printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH" + exit 1 + fi + # After verifying new installer was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +done + +# Verify MSI installer +if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then + printf "\n### ERROR: Installer MSI was not generated. Exiting...\n" + exit 1 +else + # After verifying new installer MSI was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +fi + printf "\n### Cleaning up temp files \n" rm -rf /nsm/elastic-agent-workspace -rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n" \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/ chmod 644 /nsm/elastic-fleet/so_agent-installers/* + +# if we got here all installers have been generated successfully +exit 0 diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index b62310375..21d74cf6f 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -244,11 +244,34 @@ printf '%s\n'\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State -printf "\nApplying elasticfleet state" -salt-call state.apply elasticfleet queue=True +printf "\nApplying elasticfleet state\n" +for state_attempt in {1..3}; do + if salt-call state.apply elasticfleet queue=True; then + break + elif [[ $state_attempt -lt 3 ]]; then + printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt" + sleep 10 + else + printf "\nFailure(s) in elasticfleet state... Exiting...\n" + exit 1 + fi +done # Generate installers & install Elastic Agent on the node -so-elastic-agent-gen-installers -printf "\nApplying elasticfleet.install_agent_grid state" -salt-call state.apply elasticfleet.install_agent_grid queue=True -exit 0 +for agent_gen_attempt in {1..3}; do + if so-elastic-agent-gen-installers; then + break + elif [[ $agent_gen_attempt -lt 3 ]]; then + printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt" + sleep 10 + else + printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n" + exit 1 + fi +done + +printf "\nApplying elasticfleet.install_agent_grid state\n" +if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then + printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n" + exit 1 +fi \ No newline at end of file