Merge branch 'feature/docker-prune-rework' into foxtrot

salt/common/tools/sbin/so-docker-prune

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys, argparse, re, docker
+from packaging.version import Version
+from itertools import groupby, chain
+
+
+def get_image_name(string) -> str:
+  return ':'.join(string.split(':')[:-1])
+
+
+def get_so_image_basename(string) -> str:
+  return get_image_name(string).split('/so-')[-1]
+
+
+def get_image_version(string) -> str:
+  ver = string.split(':')[-1]
+  if ver == 'latest':
+    # Version doesn't like "latest", so use a high semver
+    return '999999.9.9'
+  else:
+    return ver
+
+
+def main(quiet):
+  client = docker.from_env()
+
+  image_list = client.images.list(filters={ 'dangling': False })
+
+  # Map list of image objects to flattened list of tags (format: "name:version")
+  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
+
+  # Filter to only SO images (base name begins with "so-")
+  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
+
+  # Group tags into lists by base name (sort by same projection first)
+  tag_list.sort(key=lambda x: get_so_image_basename(x))
+  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
+
+  no_prunable = True
+  for t_list in grouped_tag_lists:
+    # Keep the 2 most current images
+    t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+    if len(t_list) <= 2:
+      continue
+    else:
+      no_prunable = False
+      for tag in t_list[2:]:
+        if not quiet: print(f'Removing image {tag}')
+        client.images.remove(tag)
+  
+  if no_prunable and not quiet:
+    print('No Security Onion images to prune')
+
+
+if __name__ == "__main__":
+  main_parser = argparse.ArgumentParser(add_help=False)
+  main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
+  args = main_parser.parse_args(sys.argv[1:])
+ 
+  main(args.quiet)

salt/docker_clean/init.sls

@@ -1,86 +1,9 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
-{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
-{% set MANAGER = salt['grains.get']('master') %}
-{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2','2.3.10','2.3.20']%}
-
-{% for VERSION in OLDVERSIONS %}
-remove_images_{{ VERSION }}:
-  docker_image.absent:
-    - force: True
-    - images:
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-acng:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-cortex:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elastalert:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet-launcher:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-grafana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-influxdb:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kratos:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-mysql:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nginx:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-redis:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soc:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soctopus:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-steno:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-frontend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-manager:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-backend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-filestream:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-telegraf:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-acng:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-thehive-cortex:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-curator:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-domainstats:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-elastalert:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-elasticsearch:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-filebeat:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-fleet:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-fleet-launcher:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-freqserver:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-grafana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-idstools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-influxdb:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-kibana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-kratos:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-logstash:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-minio:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-mysql:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-nginx:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-pcaptools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-playbook:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-redis:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-soc:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-soctopus:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-steno:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-frontend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-manager:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-backend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-filestream:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-suricata:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-telegraf:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-thehive:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-thehive-es:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-wazuh:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-zeek:{{ VERSION }}'
-{% endfor %}
+prune_images:
+  cmd.run:
+    - name: so-docker-prune
 
 {% else %}
 

setup/so-functions

@@ -2031,6 +2031,7 @@ saltify() {
 					python36-dateutil\
 					python36-m2crypto\
 					python36-mysql\
+					python36-packaging\
 					yum-utils\
 					device-mapper-persistent-data\
 					lvm2\
@@ -2119,9 +2120,9 @@ saltify() {
 		retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1
 		retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1
 		if [[ $OSVER != 'xenial' ]]; then
-			retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1
+			retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1
 		else
-			retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb"  >> "$setup_log" 2>&1 || exit 1
+			retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-packaging"  >> "$setup_log" 2>&1 || exit 1
 		fi
 	fi
 }