From 544fa824ea3b5efe9cb0c24920db8e8227a94f59 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 May 2023 14:17:59 -0400 Subject: [PATCH 1/4] Initial cut for Artifact Registry --- salt/docker/defaults.yaml | 1 + salt/elasticfleet/artifact_registry.sls | 11 +++++++++++ salt/firewall/assigned_hostgroups.map.yaml | 18 ++++++++++++++++++ salt/firewall/ports/ports.yaml | 3 +++ salt/nginx/etc/nginx.conf | 16 ++++++++++++++++ salt/nginx/init.sls | 1 + setup/so-functions | 11 +++++++++++ setup/so-setup | 2 ++ 8 files changed, 63 insertions(+) create mode 100644 salt/elasticfleet/artifact_registry.sls diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 0fb1d91b8..19dda3d35 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -54,6 +54,7 @@ docker: port_bindings: - 80:80 - 443:443 + - 8443:8443 'so-playbook': final_octet: 32 port_bindings: diff --git a/salt/elasticfleet/artifact_registry.sls b/salt/elasticfleet/artifact_registry.sls new file mode 100644 index 000000000..565bdbb46 --- /dev/null +++ b/salt/elasticfleet/artifact_registry.sls @@ -0,0 +1,11 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. + + +fleetartifactdir: + file.directory: + - name: /nsm/elastic-fleet/artifacts + - user: 947 + - group: 939 + - makedirs: True diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index b9a8f7fb2..cd75c07a1 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -46,6 +46,7 @@ role: portgroups: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} strelka_frontend: portgroups: - {{ portgroups.strelka_frontend }} @@ -74,10 +75,12 @@ role: portgroups: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} elastic_agent_endpoint: portgroups: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} INPUT: hostgroups: anywhere: @@ -117,6 +120,7 @@ role: - {{ portgroups.docker_registry }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} {% if ISAIRGAP is sameas true %} - {{ portgroups.agrules }} {% endif %} @@ -126,6 +130,7 @@ role: - {{ portgroups.beats_5644 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} @@ -140,6 +145,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} heavynodes: portgroups: - {{ portgroups.redis }} @@ -151,6 +157,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} self: portgroups: - {{ portgroups.syslog}} @@ -170,6 +177,7 @@ role: portgroups: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} endgame: portgroups: - {{ portgroups.endgame }} @@ -212,12 +220,14 @@ role: - {{ portgroups.docker_registry }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} - {{ portgroups.yum }} - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} @@ -231,6 +241,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} heavynodes: portgroups: - {{ portgroups.redis }} @@ -241,6 +252,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} self: portgroups: - {{ portgroups.syslog}} @@ -257,6 +269,7 @@ role: portgroups: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} endgame: portgroups: - {{ portgroups.endgame }} @@ -312,6 +325,7 @@ role: - {{ portgroups.elasticsearch_node }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} - {{ portgroups.endgame }} - {{ portgroups.strelka_frontend }} fleet: @@ -326,6 +340,7 @@ role: - {{ portgroups.beats_5056 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} sensors: portgroups: - {{ portgroups.docker_registry }} @@ -337,6 +352,7 @@ role: - {{ portgroups.beats_5056 }} - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} searchnodes: portgroups: - {{ portgroups.docker_registry }} @@ -371,6 +387,7 @@ role: portgroups: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} endgame: portgroups: - {{ portgroups.endgame }} @@ -529,6 +546,7 @@ role: portgroups: - {{ portgroups.elastic_agent_control }} - {{ portgroups.elastic_agent_data }} + - {{ portgroups.elastic_agent_update }} analyst: portgroups: - {{ portgroups.nginx }} diff --git a/salt/firewall/ports/ports.yaml b/salt/firewall/ports/ports.yaml index 79bdf93b4..68b93fafd 100644 --- a/salt/firewall/ports/ports.yaml +++ b/salt/firewall/ports/ports.yaml @@ -35,6 +35,9 @@ firewall: elastic_agent_data: tcp: - 5055 + elastic_agent_update: + tcp: + - 8443 endgame: tcp: - 3765 diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index e6a7f3c87..502f6302a 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -43,6 +43,22 @@ http { return 307 https://{{ GLOBALS.url_base }}$request_uri; } + server { + listen 8443; + server_name {{ GLOBALS.url_base }}; + root /opt/socore/html; + location /artifacts/ { + try_files $uri =206; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + server { listen 443 ssl http2 default_server; server_name _; diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 52d018354..833bda98a 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -96,6 +96,7 @@ so-nginx: - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/:/opt/socore/html/packages + - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %} - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro diff --git a/setup/so-functions b/setup/so-functions index f7f67dfe2..8bd738830 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -962,6 +962,17 @@ detect_os() { } +download_elastic_agent_artifacts() { + #TODO - ISO + + mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/ + + curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz + + tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/ + + } + installer_progress_loop() { local i=0 local msg="${1:-Performing background actions...}" diff --git a/setup/so-setup b/setup/so-setup index aa35a459a..0cc19d990 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -605,6 +605,8 @@ if ! [[ -f $install_opt_file ]]; then gpg_rpm_import # Create the local repo and point the box to use the local repo securityonion_repo + # Download Elastic Agent Artifacts + download_elastic_agent_artifacts # Update existing packages update_packages # Install salt From c99f19251b6b7ddd30653b5fc50a3f6b261c8dd4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 May 2023 17:04:41 -0400 Subject: [PATCH 2/4] More visibility --- setup/so-functions | 6 +++--- setup/so-setup | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 8bd738830..1a04607ea 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -965,11 +965,11 @@ detect_os() { download_elastic_agent_artifacts() { #TODO - ISO - mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/ + logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" - curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz + logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz" - tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/ + logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" } diff --git a/setup/so-setup b/setup/so-setup index 0cc19d990..654484334 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -606,6 +606,7 @@ if ! [[ -f $install_opt_file ]]; then # Create the local repo and point the box to use the local repo securityonion_repo # Download Elastic Agent Artifacts + title "Downloading Elastic Agent Artifacts" download_elastic_agent_artifacts # Update existing packages update_packages From b56486d88e904d68c9426def31cd8cd2dd293ab1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 3 May 2023 08:55:29 -0400 Subject: [PATCH 3/4] Set Elastic Agent Artifact Registry URL --- salt/common/tools/sbin/so-elastic-fleet-setup | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index edb15760f..8005def18 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -62,6 +62,15 @@ elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false" # Load Integrations for default policies so-elastic-fleet-integration-policy-load +# Set Elastic Agent Artifact Registry URL +JSON_STRING=$( jq -n \ + --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ + --arg URL "http://{{ GLOBALS.url_base }}/artifacts/" \ + '{"name":$NAME,"host":$URL,"is_default":true}' + ) + +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + ### Finalization ### # Query for Enrollment Tokens for default policies From 87a20ffedec7961817b21399122e93c75c3d467d Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 3 May 2023 10:44:46 -0400 Subject: [PATCH 4/4] Refactor Wrapper gen script --- .../sbin/so-elastic-agent-gen-installers | 35 +++++++++++++++---- setup/so-setup | 4 +-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-agent-gen-installers b/salt/common/tools/sbin/so-elastic-agent-gen-installers index 805f6152a..44c352352 100755 --- a/salt/common/tools/sbin/so-elastic-agent-gen-installers +++ b/salt/common/tools/sbin/so-elastic-agent-gen-installers @@ -17,18 +17,39 @@ FLEETHOST="{{ GLOBALS.manager_ip }}" #FLEETHOST=$1 #ENROLLMENTOKEN=$2 -CONTAINERGOOS=( "linux" "darwin" "windows" ) +TARGETOS=( "linux" "darwin" "windows" ) -#rm -rf /tmp/elastic-agent-workspace -#mkdir -p /tmp/elastic-agent-workspace +printf "\n### Get rid of any previous runs\n" +rm -rf /tmp/elastic-agent-workspace +mkdir -p /tmp/elastic-agent-workspace -for OS in "${CONTAINERGOOS[@]}" +printf "\n### Extract outer tarball and then each individual tarball/zip\n" +tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-{{ GLOBALS.so_version }}.tar.gz -C /tmp/elastic-agent-workspace/ +unzip /tmp/elastic-agent-workspace/elastic-agent-*.zip -d /tmp/elastic-agent-workspace/ +for archive in /tmp/elastic-agent-workspace/*.tar.gz +do + tar xf "$archive" -C /tmp/elastic-agent-workspace/ +done + +printf "\n### Strip out unused components" +find /tmp/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete + +printf "\n### Tar everything up again" +for OS in "${TARGETOS[@]}" +do + rm -rf /tmp/elastic-agent-workspace/elastic-agent + mv /tmp/elastic-agent-workspace/elastic-agent-*-$OS-x86_64 /tmp/elastic-agent-workspace/elastic-agent + tar -czvf /tmp/elastic-agent-workspace/$OS.tar.gz -C /tmp/elastic-agent-workspace elastic-agent +done + +printf "\n### Generate OS packages using the cleaned up tarballs" +for OS in "${TARGETOS[@]}" do - printf "\n\nGenerating $OS Installer..." - #cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz + printf "\n\n### Generating $OS Installer...\n" docker run -e CGO_ENABLED=0 -e GOOS=$OS \ --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \ + --mount type=bind,source=/tmp/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_$OS - printf "\n $OS Installer Generated..." + printf "\n### $OS Installer Generated...\n" done diff --git a/setup/so-setup b/setup/so-setup index 654484334..72549d79d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -605,11 +605,11 @@ if ! [[ -f $install_opt_file ]]; then gpg_rpm_import # Create the local repo and point the box to use the local repo securityonion_repo + # Update existing packages + update_packages # Download Elastic Agent Artifacts title "Downloading Elastic Agent Artifacts" download_elastic_agent_artifacts - # Update existing packages - update_packages # Install salt saltify # Start the master service