CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactor docker_seed_registry to eliminate duplicate logic

Browse Source
This commit is contained in:
Jason Ertel
2020-11-16 13:27:23 -05:00
parent c226c1d902
commit 2ff738a61c
2 changed files with 74 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
salt/common/tools/sbin/so-image-common
View File

@@ -19,29 +19,30 @@
IMAGEREPO=securityonion IMAGEREPO=securityonion
container_list() { container_list() {
MANAGERCHECK=so-unknown MANAGERCHECK=$1
if [ -f /etc/salt/grains ]; then if [ -z "$MANAGERCHECK" ]; then
MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') MANAGERCHECK=so-unknown
if [ -f /etc/salt/grains ]; then
MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
fi
fi fi
if [ $MANAGERCHECK == 'so-import' ]; then if [ $MANAGERCHECK == 'so-import' ]; then
TRUSTED_CONTAINERS=( \ TRUSTED_CONTAINERS=( \
"so-idstools" \
"so-nginx" \
"so-filebeat" \
"so-suricata" \
"so-soc" \
"so-elasticsearch" \ "so-elasticsearch" \
"so-filebeat" \
"so-idstools" \
"so-kibana" \ "so-kibana" \
"so-kratos" \ "so-kratos" \
"so-suricata" \ "so-nginx" \
"so-registry" \
"so-pcaptools" \ "so-pcaptools" \
"so-soc" \
"so-steno" \
"so-suricata" \
"so-zeek" ) "so-zeek" )
elif [ $MANAGERCHECK != 'so-helix' ]; then elif [ $MANAGERCHECK != 'so-helix' ]; then
TRUSTED_CONTAINERS=( \ TRUSTED_CONTAINERS=( \
"so-acng" \ "so-acng" \
"so-thehive-cortex" \
"so-curator" \ "so-curator" \
"so-domainstats" \ "so-domainstats" \
"so-elastalert" \ "so-elastalert" \
@@ -65,18 +66,19 @@ container_list() {
"so-soc" \ "so-soc" \
"so-soctopus" \ "so-soctopus" \
"so-steno" \ "so-steno" \
"so-strelka-frontend" \
"so-strelka-manager" \
"so-strelka-backend" \ "so-strelka-backend" \
"so-strelka-filestream" \ "so-strelka-filestream" \
"so-strelka-frontend" \
"so-strelka-manager" \
"so-suricata" \ "so-suricata" \
"so-telegraf" \ "so-telegraf" \
"so-thehive" \ "so-thehive" \
"so-thehive-cortex" \
"so-thehive-es" \ "so-thehive-es" \
"so-wazuh" \ "so-wazuh" \
"so-zeek" ) "so-zeek" )
else else
TRUSTED_CONTAINERS=( \ TRUSTED_CONTAINERS=( \
"so-filebeat" \ "so-filebeat" \
"so-idstools" \ "so-idstools" \
"so-logstash" \ "so-logstash" \
@@ -90,11 +92,12 @@ container_list() {
} }
update_docker_containers() { update_docker_containers() {
CURLTYPE=$1 local CURLTYPE=$1
IMAGE_TAG_SUFFIX=$2 local IMAGE_TAG_SUFFIX=$2
local PROGRESS_CALLBACK=$3
CONTAINER_REGISTRY=quay.io local CONTAINER_REGISTRY=quay.io
SIGNPATH=/root/sosigs local SIGNPATH=/root/sosigs
if [ -z "$CURLTYPE" ]; then if [ -z "$CURLTYPE" ]; then
CURLTYPE=unknown CURLTYPE=unknown
@@ -117,38 +120,44 @@ update_docker_containers() {
# Download the containers from the interwebs # Download the containers from the interwebs
for i in "${TRUSTED_CONTAINERS[@]}" for i in "${TRUSTED_CONTAINERS[@]}"
do do
if [ -z "$PROGRESS_CALLBACK" ]; then
echo "Downloading $i"
else
$PROGRESS_CALLBACK $i
fi
# Pull down the trusted docker image # Pull down the trusted docker image
echo "Downloading $i" local image=$i:$VERSION$IMAGE_TAG_SUFFIX
docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image
# Get signature # Get signature
curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "Unable to pull signature file for $i:$VERSION$IMAGE_TAG_SUFFIX" echo "Unable to pull signature file for $image"
exit 1 exit 1
fi fi
# Dump our hash values # Dump our hash values
DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX) DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image)
echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$image.txt
echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$image.txt
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "Unable to inspect $i:$VERSION$IMAGE_TAG_SUFFIX" echo "Unable to inspect $image"
exit 1 exit 1
fi fi
GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt 2>&1) GPGTEST=$(gpg --verify $SIGNPATH/$image.sig $SIGNPATH/$image.txt 2>&1)
if [[ $? -eq 0 ]]; then if [[ $? -eq 0 ]]; then
if [[ -z "$SKIP_TAGPUSH" ]]; then if [[ -z "$SKIP_TAGPUSH" ]]; then
# Tag it with the new registry destination # Tag it with the new registry destination
if [ -z "$HOSTNAME" ]; then if [ -z "$HOSTNAME" ]; then
HOSTNAME=$(hostname) HOSTNAME=$(hostname)
fi fi
docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image
docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX docker push $HOSTNAME:5000/$IMAGEREPO/$image
fi fi
else else
echo "There is a problem downloading the $i:$VERSION$IMAGE_TAG_SUFFIX image. Details: " echo "There is a problem downloading the $image image. Details: "
echo "" echo ""
echo $GPGTEST echo $GPGTEST
exit 1 exit 1

138
setup/so-functions
View File

@@ -23,6 +23,11 @@ CONTAINER_REGISTRY=quay.io
SOVERSION=$(cat ../VERSION) SOVERSION=$(cat ../VERSION)
# Duplicate stdout and stderr file descriptors for use with whiptail
# Using >&10 or >&20 will override any ancestral >> or > redirects and send
# to stdout or stderr, repsectively.
exec 10>&1 20>&2
log() { log() {
msg=$1 msg=$1
level=${2:-I} level=${2:-I}
@@ -870,116 +875,37 @@ docker_registry() {
} }
docker_seed_update() {
local name=$1
local percent_delta=1
if [ "$install_type" == 'HELIXSENSOR' ]; then
percent_delta=6
fi
((docker_seed_update_percent=docker_seed_update_percent+percent_delta))
# Backup current output descriptors and reset to normal
exec 8>&1 9>&2 1>&10 2>&20
set_progress_str "$docker_seed_update_percent" "Downloading $name"
# Restore current output descriptors and remove backups
exec 1>&8- 2>&9-
}
docker_seed_registry() { docker_seed_registry() {
local VERSION="$SOVERSION" local VERSION="$SOVERSION"
if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then
if [ "$install_type" == 'IMPORT' ]; then if [ "$install_type" == 'IMPORT' ]; then
local TRUSTED_CONTAINERS=(\ container_list 'so-import'
"so-idstools" \ elif [ "$install_type" != 'HELIXSENSOR' ]; then
"so-nginx" \ container_list 'so-helix'
"so-filebeat" \
"so-suricata" \
"so-soc" \
"so-steno" \
"so-elasticsearch" \
"so-kibana" \
"so-kratos" \
"so-suricata" \
"so-pcaptools" \
"so-zeek"
)
else else
local TRUSTED_CONTAINERS=(\ container_list
"so-nginx" \
"so-filebeat" \
"so-logstash" \
"so-idstools" \
"so-redis" \
"so-steno" \
"so-suricata" \
"so-telegraf" \
"so-zeek"
)
fi fi
if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORT' ]; then
TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \
"so-acng" \
"so-thehive-cortex" \
"so-curator" \
"so-domainstats" \
"so-elastalert" \
"so-elasticsearch" \
"so-fleet" \
"so-fleet-launcher" \
"so-freqserver" \
"so-grafana" \
"so-influxdb" \
"so-kibana" \
"so-minio" \
"so-mysql" \
"so-pcaptools" \
"so-playbook" \
"so-soc" \
"so-kratos" \
"so-soctopus" \
"so-steno" \
"so-strelka-frontend" \
"so-strelka-manager" \
"so-strelka-backend" \
"so-strelka-filestream" \
"so-thehive" \
"so-thehive-es" \
"so-wazuh"
)
fi
local percent=25
# Let's make sure we have the public key
curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import -
SIGNPATH=/root/sosigs docker_seed_update_percent=25
rm -rf $SIGNPATH update_docker_containers 'netinstall' '' 'docker_seed_update' >> "$setup_log" 2>&1
mkdir -p $SIGNPATH
if [ -z "$BRANCH" ]; then
BRANCH="master"
fi
for i in "${TRUSTED_CONTAINERS[@]}"; do
if [ "$install_type" != 'HELIXSENSOR' ]; then ((percent=percent+1)); else ((percent=percent+6)); fi
# Pull down the trusted docker image
set_progress_str "$percent" "Downloading $i:$VERSION"
{
echo "Downloading $i"
docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION
# Get signature
curl -A "netinstall/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION.sig --output $SIGNPATH/$i:$VERSION.sig
if [[ $? -ne 0 ]]; then
echo "Unable to pull signature file for $i:$VERSION"
exit 1
fi
# Dump our hash values
DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION)
echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION.txt
echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION.txt
if [[ $? -ne 0 ]]; then
echo "Unable to inspect $i"
exit 1
fi
GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION.sig $SIGNPATH/$i:$VERSION.txt 2>&1)
if [[ $? -eq 0 ]]; then
# Tag it with the new registry destination
docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
else
echo "There is a problem downloading the $i image. Details: "
echo ""
echo $GPGTEST
exit 1
fi
} >> "$setup_log" 2>&1
done
else else
tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1 tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1
rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1 rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1
@@ -1006,10 +932,10 @@ firewall_generate_templates() {
local firewall_pillar_path=$local_salt_dir/salt/firewall local firewall_pillar_path=$local_salt_dir/salt/firewall
mkdir -p "$firewall_pillar_path" mkdir -p "$firewall_pillar_path"
cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1 cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1
for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do
$default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1 $default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1
done done
} }