From 33696398eb877e8a668efcd3e08f48c5b8b6b1a1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 26 Feb 2021 18:06:07 -0500 Subject: [PATCH] Add new so-docker-prune script * Script will pull list of so- images and prune any older than most recent + last version --- salt/common/tools/sbin/so-docker-prune | 77 +++++++++++++++++ salt/common/tools/sbin/so-monitor-add | 0 .../tools/sbin/so-playbook-sigma-refresh | 0 salt/common/tools/sbin/so-raid-status | 0 salt/common/tools/sbin/so-rule | 0 salt/common/tools/sbin/so-suricata-testrule | 0 salt/docker_clean/init.sls | 83 +------------------ setup/so-functions | 5 +- 8 files changed, 83 insertions(+), 82 deletions(-) create mode 100755 salt/common/tools/sbin/so-docker-prune mode change 100644 => 100755 salt/common/tools/sbin/so-monitor-add mode change 100644 => 100755 salt/common/tools/sbin/so-playbook-sigma-refresh mode change 100644 => 100755 salt/common/tools/sbin/so-raid-status mode change 100644 => 100755 salt/common/tools/sbin/so-rule mode change 100644 => 100755 salt/common/tools/sbin/so-suricata-testrule diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune new file mode 100755 index 000000000..1b6242488 --- /dev/null +++ b/salt/common/tools/sbin/so-docker-prune @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys, argparse, re, docker +from packaging.version import Version +from itertools import groupby, chain + + +def get_image_name(string) -> str: + return ':'.join(string.split(':')[:-1]) + + +def get_so_image_basename(string) -> str: + return get_image_name(string).split('/so-')[-1] + + +def get_image_version(string) -> str: + ver = string.split(':')[-1] + if ver == 'latest': + # Version doesn't like "latest", so use a high semver + return '999999.9.9' + else: + return ver + + +def main(quiet): + client = docker.from_env() + + # Get list of non-dangling images + image_list = client.images.list(filters={ 'dangling': False }) + + # Map image objects to flattened list of tags (format: "name:version") + tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list)))) + + # Filter to only SO images (base name begins with "so-") + tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list)) + + # Group tags into lists by base name (sort by same projection first) + tag_list.sort(key=lambda x: get_so_image_basename(x)) + grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ] + + no_prunable = True + for t_list in grouped_tag_lists: + # Keep the 2 most current images + t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True) + if len(t_list) <= 2: + continue + else: + no_prunable = False + for tag in t_list[2:]: + if not quiet: print(f'Removing image {tag}') + client.images.remove(tag) + + if no_prunable and not quiet: + print('No Security Onion images to prune') + + +if __name__ == "__main__": + main_parser = argparse.ArgumentParser(add_help=False) + main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False) + args = main_parser.parse_args(sys.argv[1:]) + + main(args.quiet) \ No newline at end of file diff --git a/salt/common/tools/sbin/so-monitor-add b/salt/common/tools/sbin/so-monitor-add old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-playbook-sigma-refresh b/salt/common/tools/sbin/so-playbook-sigma-refresh old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-suricata-testrule b/salt/common/tools/sbin/so-suricata-testrule old mode 100644 new mode 100755 diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index 006d108eb..a92d3aedd 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -1,86 +1,9 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} -{% set MANAGER = salt['grains.get']('master') %} -{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2','2.3.10','2.3.20']%} - -{% for VERSION in OLDVERSIONS %} -remove_images_{{ VERSION }}: - docker_image.absent: - - force: True - - images: - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-acng:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-cortex:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elastalert:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet-launcher:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-grafana:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-influxdb:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kratos:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-mysql:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nginx:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-redis:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soc:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soctopus:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-steno:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-frontend:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-manager:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-backend:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-filestream:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-telegraf:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}' - - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-acng:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-thehive-cortex:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-curator:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-domainstats:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-elastalert:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-elasticsearch:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-filebeat:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-fleet:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-fleet-launcher:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-freqserver:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-grafana:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-idstools:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-influxdb:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-kibana:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-kratos:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-logstash:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-minio:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-mysql:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-nginx:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-pcaptools:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-playbook:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-redis:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-soc:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-soctopus:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-steno:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-frontend:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-manager:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-backend:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-strelka-filestream:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-suricata:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-telegraf:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-thehive:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-thehive-es:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-wazuh:{{ VERSION }}' - - '{{ MANAGER }}:5000/securityonion/so-zeek:{{ VERSION }}' -{% endfor %} +prune_images: + cmd.run: + - name: so-docker-prune {% else %} diff --git a/setup/so-functions b/setup/so-functions index 0ad993d3d..30075bb0c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1957,6 +1957,7 @@ saltify() { python36-dateutil\ python36-m2crypto\ python36-mysql\ + python36-packaging\ yum-utils\ device-mapper-persistent-data\ lvm2\ @@ -2045,9 +2046,9 @@ saltify() { retry 50 10 "apt-get -y install salt-minion=3002.2+ds-1 salt-common=3002.2+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1 else - retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-packaging" >> "$setup_log" 2>&1 || exit 1 fi fi }