diff --git a/salt/common/tools/sbin/so-elastic-agent-gen-installers b/salt/common/tools/sbin/so-elastic-agent-gen-installers index 805f6152a..44c352352 100755 --- a/salt/common/tools/sbin/so-elastic-agent-gen-installers +++ b/salt/common/tools/sbin/so-elastic-agent-gen-installers @@ -17,18 +17,39 @@ FLEETHOST="{{ GLOBALS.manager_ip }}" #FLEETHOST=$1 #ENROLLMENTOKEN=$2 -CONTAINERGOOS=( "linux" "darwin" "windows" ) +TARGETOS=( "linux" "darwin" "windows" ) -#rm -rf /tmp/elastic-agent-workspace -#mkdir -p /tmp/elastic-agent-workspace +printf "\n### Get rid of any previous runs\n" +rm -rf /tmp/elastic-agent-workspace +mkdir -p /tmp/elastic-agent-workspace -for OS in "${CONTAINERGOOS[@]}" +printf "\n### Extract outer tarball and then each individual tarball/zip\n" +tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-{{ GLOBALS.so_version }}.tar.gz -C /tmp/elastic-agent-workspace/ +unzip /tmp/elastic-agent-workspace/elastic-agent-*.zip -d /tmp/elastic-agent-workspace/ +for archive in /tmp/elastic-agent-workspace/*.tar.gz +do + tar xf "$archive" -C /tmp/elastic-agent-workspace/ +done + +printf "\n### Strip out unused components" +find /tmp/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete + +printf "\n### Tar everything up again" +for OS in "${TARGETOS[@]}" +do + rm -rf /tmp/elastic-agent-workspace/elastic-agent + mv /tmp/elastic-agent-workspace/elastic-agent-*-$OS-x86_64 /tmp/elastic-agent-workspace/elastic-agent + tar -czvf /tmp/elastic-agent-workspace/$OS.tar.gz -C /tmp/elastic-agent-workspace elastic-agent +done + +printf "\n### Generate OS packages using the cleaned up tarballs" +for OS in "${TARGETOS[@]}" do - printf "\n\nGenerating $OS Installer..." - #cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz + printf "\n\n### Generating $OS Installer...\n" docker run -e CGO_ENABLED=0 -e GOOS=$OS \ --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \ + --mount type=bind,source=/tmp/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_$OS - printf "\n $OS Installer Generated..." + printf "\n### $OS Installer Generated...\n" done diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index edb15760f..8005def18 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -62,6 +62,15 @@ elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false" # Load Integrations for default policies so-elastic-fleet-integration-policy-load +# Set Elastic Agent Artifact Registry URL +JSON_STRING=$( jq -n \ + --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ + --arg URL "http://{{ GLOBALS.url_base }}/artifacts/" \ + '{"name":$NAME,"host":$URL,"is_default":true}' + ) + +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + ### Finalization ### # Query for Enrollment Tokens for default policies diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 0fb1d91b8..19dda3d35 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -54,6 +54,7 @@ docker: port_bindings: - 80:80 - 443:443 + - 8443:8443 'so-playbook': final_octet: 32 port_bindings: diff --git a/salt/elasticfleet/artifact_registry.sls b/salt/elasticfleet/artifact_registry.sls new file mode 100644 index 000000000..565bdbb46 --- /dev/null +++ b/salt/elasticfleet/artifact_registry.sls @@ -0,0 +1,11 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. + + +fleetartifactdir: + file.directory: + - name: /nsm/elastic-fleet/artifacts + - user: 947 + - group: 939 + - makedirs: True diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index e31d9f1ed..1526e5504 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -78,6 +78,10 @@ firewall: tcp: - 5055 udp: [] + elastic_agent_update: + tcp: + - 8443 + udp: [] endgame: tcp: - 3765 @@ -205,6 +209,7 @@ firewall: portgroups: - elastic_agent_control - elastic_agent_data + - elastic_agent_update strelka_frontend: portgroups: - strelka_frontend @@ -273,10 +278,12 @@ firewall: portgroups: - elastic_agent_control - elastic_agent_data + - elastic_agent_update elastic_agent_endpoint: portgroups: - elastic_agent_control - elastic_agent_data + - elastic_agent_update customhostgroup0: portgroups: [] customhostgroup1: @@ -356,12 +363,14 @@ firewall: - docker_registry - elastic_agent_control - elastic_agent_data + - elastic_agent_update sensor: portgroups: - beats_5044 - beats_5644 - elastic_agent_control - elastic_agent_data + - elastic_agent_update - yum - docker_registry - influxdb @@ -376,6 +385,7 @@ firewall: - influxdb - elastic_agent_control - elastic_agent_data + - elastic_agent_update heavynode: portgroups: - redis @@ -387,6 +397,7 @@ firewall: - influxdb - elastic_agent_control - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog @@ -406,6 +417,7 @@ firewall: portgroups: - elastic_agent_control - elastic_agent_data + - elastic_agent_update endgame: portgroups: - endgame @@ -488,12 +500,14 @@ firewall: - docker_registry - elastic_agent_control - elastic_agent_data + - elastic_agent_update sensor: portgroups: - beats_5044 - beats_5644 - elastic_agent_control - elastic_agent_data + - elastic_agent_update - yum - docker_registry - influxdb @@ -507,6 +521,7 @@ firewall: - influxdb - elastic_agent_control - elastic_agent_data + - elastic_agent_update heavynode: portgroups: - redis @@ -517,6 +532,7 @@ firewall: - influxdb - elastic_agent_control - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog @@ -533,6 +549,7 @@ firewall: portgroups: - elastic_agent_control - elastic_agent_data + - elastic_agent_update endgame: portgroups: - endgame @@ -628,6 +645,7 @@ firewall: - elasticsearch_node - elastic_agent_control - elastic_agent_data + - elastic_agent_update - endgame - strelka_frontend fleet: @@ -642,6 +660,7 @@ firewall: - beats_5056 - elastic_agent_control - elastic_agent_data + - elastic_agent_update sensor: portgroups: - docker_registry @@ -653,6 +672,7 @@ firewall: - beats_5056 - elastic_agent_control - elastic_agent_data + - elastic_agent_update searchnode: portgroups: - docker_registry @@ -687,6 +707,7 @@ firewall: portgroups: - elastic_agent_control - elastic_agent_data + - elastic_agent_update endgame: portgroups: - endgame @@ -1005,6 +1026,7 @@ firewall: portgroups: - elastic_agent_control - elastic_agent_data + - elastic_agent_update analyst: portgroups: - nginx diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 02209af60..9954c1305 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -106,6 +106,9 @@ firewall: elastic_agent_data: tcp: *tcpsettings udp: *udpsettings + elastic_agent_update: + tcp: *tcpsettings + udp: *udpsettings endgame: tcp: *tcpsettings udp: *udpsettings diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index e6a7f3c87..502f6302a 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -43,6 +43,22 @@ http { return 307 https://{{ GLOBALS.url_base }}$request_uri; } + server { + listen 8443; + server_name {{ GLOBALS.url_base }}; + root /opt/socore/html; + location /artifacts/ { + try_files $uri =206; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + server { listen 443 ssl http2 default_server; server_name _; diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 52d018354..833bda98a 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -96,6 +96,7 @@ so-nginx: - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/:/opt/socore/html/packages + - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %} - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro diff --git a/setup/so-functions b/setup/so-functions index d0924fc11..4fbd26c41 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -964,6 +964,17 @@ detect_os() { } +download_elastic_agent_artifacts() { + #TODO - ISO + + logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" + + logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" + + logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" + + } + installer_progress_loop() { local i=0 local msg="${1:-Performing background actions...}" diff --git a/setup/so-setup b/setup/so-setup index aa35a459a..72549d79d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -607,6 +607,9 @@ if ! [[ -f $install_opt_file ]]; then securityonion_repo # Update existing packages update_packages + # Download Elastic Agent Artifacts + title "Downloading Elastic Agent Artifacts" + download_elastic_agent_artifacts # Install salt saltify # Start the master service