From 9183c0a92ccb852af6943da02d9d43f1ee8cbeba Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 19 Feb 2021 09:24:12 -0500 Subject: [PATCH 01/24] [feat] Initial so-rules script * Quote curly braces in minion pillar, need to add sed function in soup --- salt/common/tools/sbin/so-rules | 426 ++++++++++++++++++++++++++++++++ setup/so-functions | 6 +- 2 files changed, 429 insertions(+), 3 deletions(-) create mode 100644 salt/common/tools/sbin/so-rules diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules new file mode 100644 index 000000000..6db261bda --- /dev/null +++ b/salt/common/tools/sbin/so-rules @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Local exit codes: + - General error: 1 + - Invalid argument: 2 + - File error: 3 +""" + +import sys, os, subprocess, argparse, signal +import copy +import re +import textwrap +import yaml + +minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' +salt_proc: subprocess.CompletedProcess = None + + +def print_err(string: str): + print(string, file=sys.stderr) + + +def check_apply(args: dict): + if args.apply: + print('Applying idstools state...') + return subprocess.run(['salt-call', 'state.apply', 'idstools', 'queue=True']) + else: + return 0 + + +def find_minion_pillar(minion_id: str) -> str: + if minion_id == None: + regex = '^.*_(manager|standalone)\.sls$' + else: + regex = f'^{minion_id}\.sls$' + + result = [] + for root, _, files in os.walk(minion_pillar_dir): + for f_minion_id in files: + if re.search(regex, f_minion_id): + result.append(os.path.join(root, f_minion_id)) + + if len(result) == 0: + if minion_id == None: + print_err('Could not find minion pillar with minion id matching *_manager or *_standalone') + sys.exit(3) + else: + print_err(f'Could not find minion pillar for minion id: {minion_id}') + sys.exit(3) + elif len(result) > 1: + res_str = ', '.join(f'\"{result}\"') + if minion_id == None: + print_err('(This should not happen, the system is in an error state if you see this message.)') + print_err('More than one manager or standalone pillar exists, minion id\'s listed below:') + print_err(f' {res_str}') + sys.exit(3) + else: + print_err(f'Multiple minion pillars matched the minion id {minion_id}. Are you sure this is a complete minion id?') + sys.exit(3) + else: + return result[0] + + +def read_pillar(pillar: str): + try: + with open(pillar, 'r') as f: + loaded_yaml = yaml.safe_load(f.read()) + if loaded_yaml is None: + print_err(f'Could not parse {pillar}') + sys.exit(3) + return loaded_yaml + except: + print_err(f'Could not open {pillar}') + sys.exit(3) + + +def write_pillar(pillar: str, content: dict): + try: + sids = content['idstools']['sids'] + if sids['disabled'] is not None: + if len(sids['disabled']) == 0: sids['disabled'] = None + if sids['enabled'] is not None: + if len(sids['enabled']) == 0: sids['enabled'] = None + if sids['modify'] is not None: + if len(sids['modify']) == 0: sids['modify'] = None + + with open(pillar, 'w') as f: + return yaml.dump(content, f, default_flow_style=False) + except Exception as e: + print_err(f'Could not open {pillar}') + sys.exit(3) + + +def check_sid_pattern(sid_pattern: str, sid_only: bool = False): + message = f'SID {sid_pattern} is not valid, did you forget the \"re:\" prefix for a regex pattern?' + + if sid_pattern.startswith('re:') and not sid_only: + r_string = sid_pattern[3:] + if not valid_regex(r_string): + print_err('Invalid regex pattern.') + return False + else: + return True + elif sid_pattern == '*': + return True + else: + sid: int + try: + sid = int(sid_pattern.replace('*', '')) + except: + print_err(message) + return False + + if sid >= 0: + return True + else: + print_err(message) + return False + + +def valid_regex(pattern: str): + try: + re.compile(pattern) + return True + except re.error: + return False + + +def sids_key_exists(pillar: dict, key: str): + return key in pillar.get('idstools', {}).get('sids', {}) + + +def rem_from_sids(pillar: dict, key: str, val: str, optional = False): + pillar_dict = copy.deepcopy(pillar) + arr = pillar_dict['idstools']['sids'][key] + if arr is None or val not in arr: + if not optional: print(f'{val} already does not exist in {key}') + else: + pillar_dict['idstools']['sids'][key].remove(val) + return pillar_dict + + +def add_to_sids(pillar: dict, key: str, val: str, optional = False): + pillar_dict = copy.deepcopy(pillar) + if pillar_dict['idstools']['sids'][key] is None: + pillar_dict['idstools']['sids'][key] = [] + if val in pillar_dict['idstools']['sids'][key]: + if not optional: print(f'{val} already exists in {key}') + else: + pillar_dict['idstools']['sids'][key].append(val) + return pillar_dict + + +def add_rem_disabled(args: dict): + global salt_proc + + if not check_sid_pattern(args.sid_pattern): + return 2 + + pillar_dict = read_pillar(args.pillar) + + if args.remove: + temp_pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern) + else: + temp_pillar_dict = add_to_sids(pillar_dict, 'disabled', args.sid_pattern) + + if temp_pillar_dict['idstools']['sids']['disabled'] == pillar_dict['idstools']['sids']['disabled']: + salt_proc = check_apply(args) + return salt_proc + else: + pillar_dict = temp_pillar_dict + + if not args.remove: + if sids_key_exists(pillar_dict, 'enabled'): + pillar_dict = rem_from_sids(pillar_dict, 'enabled', args.sid_pattern, optional=True) + + modify = pillar_dict.get('idstools', {}).get('sids', {}).get('modify') + if modify is not None: + rem_candidates = [] + for action in modify: + if action.startswith(f'{args.sid_pattern} '): + rem_candidates.append(action) + if len(rem_candidates) > 0: + for item in rem_candidates: + print(f' - {item}') + answer = input(f'The above modify actions contain {args.sid_pattern}. Would you like to remove them? (Y/n) ') + while answer.lower() not in [ 'y', 'n', '' ]: + for item in rem_candidates: + print(f' - {item}') + answer = input(f'The above modify actions contain {args.sid_pattern}. Would you like to remove them? (Y/n) ') + if answer.lower() in [ 'y', '' ]: + for item in rem_candidates: + modify.remove(item) + pillar_dict['idstools']['sids']['modify'] = modify + + write_pillar(pillar=args.pillar, content=pillar_dict) + + salt_proc = check_apply(args) + return salt_proc + + +def list_disabled_rules(args: dict): + pillar_dict = read_pillar(args.pillar) + + disabled = pillar_dict.get('idstools', {}).get('sids', {}).get('disabled') + if disabled is None: + print('No rules disabled.') + return 0 + else: + print('Disabled rules:') + for rule in disabled: + print(f' - {rule}') + return 0 + + +def add_rem_enabled(args: dict): + global salt_proc + + if not check_sid_pattern(args.sid_pattern): + return 2 + + pillar_dict = read_pillar(args.pillar) + + if args.remove: + temp_pillar_dict = rem_from_sids(pillar_dict, 'enabled', args.sid_pattern) + else: + temp_pillar_dict = add_to_sids(pillar_dict, 'enabled', args.sid_pattern) + + if temp_pillar_dict == pillar_dict: + salt_proc = check_apply(args) + return salt_proc + else: + pillar_dict = temp_pillar_dict + + if not args.remove: + if sids_key_exists(pillar_dict, 'disabled'): + pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern, optional=True) + + write_pillar(pillar=args.pillar, content=pillar_dict) + + salt_proc = check_apply(args) + return salt_proc + + +def list_enabled_rules(args: dict): + pillar_dict = read_pillar(args.pillar) + + enabled = pillar_dict.get('idstools', {}).get('sids', {}).get('enabled') + if enabled is None: + print('No rules explicitely enabled.') + return 0 + else: + print('Enabled rules:') + for rule in enabled: + print(f' - {rule}') + return 0 + + +def add_rem_modify(args: dict): + global salt_proc + + if not check_sid_pattern(args.sid_pattern): + return 2 + + if not valid_regex(args.search_term): + print_err('Search term is not a valid regex pattern.') + + string_val = f'{args.sid_pattern} \"{args.search_term}\" \"{args.replace_term}\"' + + pillar_dict = read_pillar(args.pillar) + + if args.remove: + temp_pillar_dict = rem_from_sids(pillar_dict, 'modify', string_val) + else: + temp_pillar_dict = add_to_sids(pillar_dict, 'modify', string_val) + + if temp_pillar_dict == pillar_dict: + salt_proc = check_apply(args) + return salt_proc + else: + pillar_dict = temp_pillar_dict + + # TODO: Determine if a rule should be removed from disabled if modified. + if not args.remove: + if sids_key_exists(pillar_dict, 'disabled'): + pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern, optional=True) + + write_pillar(pillar=args.pillar, content=pillar_dict) + + salt_proc = check_apply(args) + return salt_proc + + +def list_modified_rules(args: dict): + pillar_dict = read_pillar(args.pillar) + + modify = pillar_dict.get('idstools', {}).get('sids', {}).get('modify') + if modify is None: + print('No rules currently modified.') + return 0 + else: + print('Modified rules + modifications:') + for rule in modify: + print(f' - {rule}') + return 0 + + +def sigint_handler(*_): + print('Exiting gracefully on Ctrl-C') + if salt_proc is not None: salt_proc.send_signal(signal.SIGINT) + sys.exit(0) + + +def main(): + signal.signal(signal.SIGINT, sigint_handler) + + if os.geteuid() != 0: + print_err('You must run this script as root') + sys.exit(1) + + main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) + main_parser.add_argument('--apply', + action='store_const', + const=True, + required=False, + help="After updating rule configuration, apply the idstools state.") + main_parser.add_argument('--minion', + dest='minion_id', + required=False, + help='Defaults to manager (i.e. action applied to entire grid).') + + subcommand_desc = textwrap.dedent( + """\ + disabled Manage and list disabled rules (add, remove, list) + enabled Manage and list enabled rules (add, remove, list) + modify Manage and list modified rules (add, remove, list) + """ + ) + subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='') + + + # Disabled actions + disabled = subparsers.add_parser('disabled') + disabled_sub = disabled.add_subparsers() + + disabled_add = disabled_sub.add_parser('add') + disabled_add.set_defaults(func=add_rem_disabled) + disabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + + disabled_rem = disabled_sub.add_parser('remove') + disabled_rem.set_defaults(func=add_rem_disabled, remove=True) + disabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + + disabled_list = disabled_sub.add_parser('list') + disabled_list.set_defaults(func=list_disabled_rules) + + + # Enabled actions + enabled = subparsers.add_parser('enabled') + enabled_sub = enabled.add_subparsers() + + enabled_add = enabled_sub.add_parser('add') + enabled_add.set_defaults(func=add_rem_enabled) + enabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + + enabled_rem = enabled_sub.add_parser('remove') + enabled_rem.set_defaults(func=add_rem_enabled, remove=True) + enabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + + enabled_list = enabled_sub.add_parser('list') + enabled_list.set_defaults(func=list_enabled_rules) + + + # Modify actions + modify = subparsers.add_parser('modify') + modify_sub = modify.add_subparsers() + + modify_add = modify_sub.add_parser('add') + modify_add.set_defaults(func=add_rem_modify) + modify_add.add_argument('sid_pattern', metavar='SID', help='A valid SID (ex: "4321").') + modify_add.add_argument('search_term', metavar='SEARCH_TERM', help='A quoted regex search term (ex: "\$EXTERNAL_NET")') + modify_add.add_argument('replace_term', metavar='REPLACE_TERM', help='The text to replace the search term with') + + modify_rem = modify_sub.add_parser('remove') + modify_rem.set_defaults(func=add_rem_modify, remove=True) + modify_rem.add_argument('sid_pattern', metavar='SID', help='A valid SID (ex: "4321").') + modify_rem.add_argument('search_term', metavar='SEARCH_TERM', help='A quoted regex search term (ex: "\$EXTERNAL_NET")') + modify_rem.add_argument('replace_term', metavar='REPLACE_TERM', help='The text to replace the search term with') + + modify_list = modify_sub.add_parser('list') + modify_list.set_defaults(func=list_modified_rules) + + + # Begin parse + run + args = main_parser.parse_args(sys.argv[1:]) + + if not hasattr(args, 'remove'): + args.remove = False + + args.pillar = find_minion_pillar(args.minion_id) + + exit_code = args.func(args) + if isinstance(exit_code, subprocess.CompletedProcess): + sys.exit(exit_code.returncode) + else: + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/setup/so-functions b/setup/so-functions index 6eb2bc1ed..c1064f782 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1168,7 +1168,7 @@ elasticsearch_pillar() { " esclustername: $ESCLUSTERNAME" >> "$pillar_file" else printf '%s\n'\ - " esclustername: {{ grains.host }}" >> "$pillar_file" + " esclustername: '{{ grains.host }}'" >> "$pillar_file" fi printf '%s\n'\ " node_type: '$NODETYPE'"\ @@ -1429,7 +1429,7 @@ manager_pillar() { " mainip: '$MAINIP'"\ " mainint: '$MNIC'"\ " esheap: '$ES_HEAP_SIZE'"\ - " esclustername: {{ grains.host }}"\ + " esclustername: '{{ grains.host }}'"\ " freq: 0"\ " domainstats: 0" >> "$pillar_file" @@ -1453,7 +1453,7 @@ manager_pillar() { " mainip: '$MAINIP'"\ " mainint: '$MNIC'"\ " esheap: '$NODE_ES_HEAP_SIZE'"\ - " esclustername: {{ grains.host }}"\ + " esclustername: '{{ grains.host }}'"\ " node_type: '$NODETYPE'"\ " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ From 2184c6d59fb92ccebed3f072a0c78a13efa85eba Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 19 Feb 2021 09:31:22 -0500 Subject: [PATCH 02/24] [fix] Create dict value if it doesn't exist --- salt/common/tools/sbin/so-rules | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index 6db261bda..0406eb67a 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -174,6 +174,9 @@ def add_rem_disabled(args: dict): return 2 pillar_dict = read_pillar(args.pillar) + + if not sids_key_exists(pillar_dict, 'disabled'): + pillar_dict['idstools']['sids']['disabled'] = None if args.remove: temp_pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern) @@ -237,6 +240,9 @@ def add_rem_enabled(args: dict): pillar_dict = read_pillar(args.pillar) + if not sids_key_exists(pillar_dict, 'enabled'): + pillar_dict['idstools']['sids']['enabled'] = None + if args.remove: temp_pillar_dict = rem_from_sids(pillar_dict, 'enabled', args.sid_pattern) else: @@ -285,6 +291,9 @@ def add_rem_modify(args: dict): pillar_dict = read_pillar(args.pillar) + if not sids_key_exists(pillar_dict, 'modify'): + pillar_dict['idstools']['sids']['modify'] = None + if args.remove: temp_pillar_dict = rem_from_sids(pillar_dict, 'modify', string_val) else: From 4689e32ce48bf64589da12d86ad56866276a84f2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 19 Feb 2021 10:18:06 -0500 Subject: [PATCH 03/24] Add sed for curly braces in minion pillars to soup --- salt/common/tools/sbin/soup | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 09be9531c..0c322742d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -205,7 +205,8 @@ pillar_changes() { [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 - [[ "$INSTALLEDVERSION" == 2.3.0 ]] || [[ "$INSTALLEDVERSION" == 2.3.1 ]] || [[ "$INSTALLEDVERSION" == 2.3.2 ]] || [[ "$INSTALLEDVERSION" == 2.3.10 ]] && 2.3.0_to_2.3.20 + [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && fun_2.3.0_to_2.3.20 + [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && fun_2.3.20_to_2.3.30 } rc1_to_rc2() { @@ -291,7 +292,7 @@ rc3_to_2.3.0() { INSTALLEDVERSION=2.3.0 } -2.3.0_to_2.3.20(){ +fun_2.3.0_to_2.3.20(){ DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 # Remove PCAP from global sed '/pcap:/d' /opt/so/saltstack/local/pillar/global.sls @@ -330,6 +331,14 @@ rc3_to_2.3.0() { } +fun_2.3.20_to_2.3.30() { + # Replace any curly brace scalars with the same scalar in single quotes + readarray -t minion_pillars <<< "$(find /opt/so/saltstack/local/minions -type f -name '*.sls')" + for pillar in "${minion_pillars[@]}"; do + sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar" + done +} + space_check() { # Check to see if there is enough space CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') From 34174a32904737139be8c7fdfd3d660c2a20708c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 19 Feb 2021 14:34:32 -0500 Subject: [PATCH 04/24] Print relevant help if no/partial command passed --- salt/common/tools/sbin/so-rules | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index 0406eb67a..ee9857bff 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -361,7 +361,7 @@ def main(): modify Manage and list modified rules (add, remove, list) """ ) - subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='') + subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command') # Disabled actions @@ -424,7 +424,21 @@ def main(): args.pillar = find_minion_pillar(args.minion_id) + + if hasattr(args, 'func'): exit_code = args.func(args) + else: + if args.command is None: + main_parser.print_help() + else: + if args.command == 'disabled': + disabled.print_help() + elif args.command == 'enabled': + enabled.print_help() + elif args.command == 'modify': + modify.print_help() + sys.exit(0) + if isinstance(exit_code, subprocess.CompletedProcess): sys.exit(exit_code.returncode) else: From c73970620d5a644ba823ab1453c68e6a451210ab Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 19 Feb 2021 14:38:43 -0500 Subject: [PATCH 05/24] [fix] Correct indent --- salt/common/tools/sbin/so-rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index ee9857bff..56e9f8fff 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -426,7 +426,7 @@ def main(): if hasattr(args, 'func'): - exit_code = args.func(args) + exit_code = args.func(args) else: if args.command is None: main_parser.print_help() From abae67356816840bc656916281950c156c2effef Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 10:00:29 -0500 Subject: [PATCH 06/24] Update help text to reflect arg requirement changes --- salt/common/tools/sbin/so-rules | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index 56e9f8fff..154ecf461 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -364,17 +364,19 @@ def main(): subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command') + sid_or_regex_help = 'A valid SID with optional wildcard (ex: "4321" or "432*") or a regular expression pattern (ex: "re:heartbleed|spectre")' + # Disabled actions disabled = subparsers.add_parser('disabled') disabled_sub = disabled.add_subparsers() disabled_add = disabled_sub.add_parser('add') disabled_add.set_defaults(func=add_rem_disabled) - disabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + disabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) disabled_rem = disabled_sub.add_parser('remove') disabled_rem.set_defaults(func=add_rem_disabled, remove=True) - disabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + disabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) disabled_list = disabled_sub.add_parser('list') disabled_list.set_defaults(func=list_disabled_rules) @@ -386,31 +388,34 @@ def main(): enabled_add = enabled_sub.add_parser('add') enabled_add.set_defaults(func=add_rem_enabled) - enabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + enabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) enabled_rem = enabled_sub.add_parser('remove') enabled_rem.set_defaults(func=add_rem_enabled, remove=True) - enabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help='A valid SID (ex: "4321") or a regular expression pattern (ex: "re:heartbleed|spectre")') + enabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) enabled_list = enabled_sub.add_parser('list') enabled_list.set_defaults(func=list_enabled_rules) + search_term_help='A quoted regex search term (ex: "\$EXTERNAL_NET")' + replace_term_help='The text to replace the search term with' + # Modify actions modify = subparsers.add_parser('modify') modify_sub = modify.add_subparsers() modify_add = modify_sub.add_parser('add') modify_add.set_defaults(func=add_rem_modify) - modify_add.add_argument('sid_pattern', metavar='SID', help='A valid SID (ex: "4321").') - modify_add.add_argument('search_term', metavar='SEARCH_TERM', help='A quoted regex search term (ex: "\$EXTERNAL_NET")') - modify_add.add_argument('replace_term', metavar='REPLACE_TERM', help='The text to replace the search term with') + modify_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) + modify_add.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help) + modify_add.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help) modify_rem = modify_sub.add_parser('remove') modify_rem.set_defaults(func=add_rem_modify, remove=True) - modify_rem.add_argument('sid_pattern', metavar='SID', help='A valid SID (ex: "4321").') - modify_rem.add_argument('search_term', metavar='SEARCH_TERM', help='A quoted regex search term (ex: "\$EXTERNAL_NET")') - modify_rem.add_argument('replace_term', metavar='REPLACE_TERM', help='The text to replace the search term with') + modify_rem.add_argument('sid_pattern', metavar='SID', help=sid_or_regex_help) + modify_rem.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help) + modify_rem.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help) modify_list = modify_sub.add_parser('list') modify_list.set_defaults(func=list_modified_rules) From 3467f30603bab452efc912904c0ca16f8d084a0d Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 22 Feb 2021 10:27:24 -0500 Subject: [PATCH 07/24] Improve support for Suricata metadata #2200 --- .../files/ingest/{zeek.dns.tld => dns.tld} | 0 salt/elasticsearch/files/ingest/suricata.dns | 25 +++++++------- .../files/ingest/suricata.fileinfo | 13 +++---- salt/elasticsearch/files/ingest/suricata.tls | 34 +++++++++---------- salt/elasticsearch/files/ingest/zeek.dns | 2 +- 5 files changed, 38 insertions(+), 36 deletions(-) rename salt/elasticsearch/files/ingest/{zeek.dns.tld => dns.tld} (100%) diff --git a/salt/elasticsearch/files/ingest/zeek.dns.tld b/salt/elasticsearch/files/ingest/dns.tld similarity index 100% rename from salt/elasticsearch/files/ingest/zeek.dns.tld rename to salt/elasticsearch/files/ingest/dns.tld diff --git a/salt/elasticsearch/files/ingest/suricata.dns b/salt/elasticsearch/files/ingest/suricata.dns index a40107819..e0986c97c 100644 --- a/salt/elasticsearch/files/ingest/suricata.dns +++ b/salt/elasticsearch/files/ingest/suricata.dns @@ -2,18 +2,19 @@ "description" : "suricata.dns", "processors" : [ { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.type", "target_field": "dns.type", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.tx_id", "target_field": "dns.id", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.version", "target_field": "dns.version", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.rrname", "target_field": "dns.query.name", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.flags", "target_field": "dns.flags", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.type", "target_field": "dns.type", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.tx_id", "target_field": "dns.id", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.version", "target_field": "dns.version", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rrname", "target_field": "dns.query.name", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.flags", "target_field": "dns.flags", "ignore_missing": true } }, { "rename": { "field": "message2.dns.qr", "target_field": "dns.qr", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.rd", "target_field": "dns.recursion.desired", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.ra", "target_field": "dns.recursion.available", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.rcode", "target_field": "dns.response.code", "ignore_missing": true } }, - { "rename": { "field": "message2.grouped.A", "target_field": "dns.answers.data", "ignore_missing": true } }, - { "rename": { "field": "message2.grouped.CNAME", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rd", "target_field": "dns.recursion.desired", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.ra", "target_field": "dns.recursion.available", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rcode", "target_field": "dns.response.code", "ignore_missing": true } }, + { "rename": { "field": "message2.grouped.A", "target_field": "dns.answers.data", "ignore_missing": true } }, + { "rename": { "field": "message2.grouped.CNAME", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "common" } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/suricata.fileinfo b/salt/elasticsearch/files/ingest/suricata.fileinfo index 7b5bff14c..d5147fb40 100644 --- a/salt/elasticsearch/files/ingest/suricata.fileinfo +++ b/salt/elasticsearch/files/ingest/suricata.fileinfo @@ -2,17 +2,18 @@ "description" : "suricata.fileinfo", "processors" : [ { "set": { "field": "dataset", "value": "file" } }, - { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.fileinfo.filename", "target_field": "file.name", "ignore_missing": true } }, - { "rename": { "field": "message2.fileinfo.gaps", "target_field": "file.bytes.missing", "ignore_missing": true } }, - { "rename": { "field": "message2.fileinfo.magic", "target_field": "file.mime_type", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.filename", "target_field": "file.name", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.gaps", "target_field": "file.bytes.missing", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.magic", "target_field": "file.mime_type", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.md5", "target_field": "hash.md5", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.sha1", "target_field": "hash.sha1", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.sid", "target_field": "rule.uuid", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.size", "target_field": "file.size", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.state", "target_field": "file.state", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.stored", "target_field": "file.saved", "ignore_missing": true } }, + { "set": { "if": "ctx.network?.protocol != null", "field": "file.source", "value": "{{network.protocol}}" } }, { "pipeline": { "name": "common" } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/suricata.tls b/salt/elasticsearch/files/ingest/suricata.tls index 0dfc06eaa..6fb0aa5ad 100644 --- a/salt/elasticsearch/files/ingest/suricata.tls +++ b/salt/elasticsearch/files/ingest/suricata.tls @@ -1,22 +1,22 @@ { "description" : "suricata.tls", "processors" : [ - { "set": { "field": "dataset", "value": "ssl" } }, - { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.subject", "target_field": "ssl.certificate.subject", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.serial", "target_field": "ssl.certificate.serial", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.fingerprint", "target_field": "ssl.certificate.fingerprint", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.version", "target_field": "ssl.certificate.version", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3.hash", "target_field": "hash.ja3", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3.hash.string", "target_field": "hash.ja3_string", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3s.hash", "target_field": "hash.ja3s", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3s.hash.string", "target_field": "hash.ja3s_string", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.notbefore", "target_field": "x509.certificate.not_valid_before", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.notafter", "target_field": "x509.certificate.not_valid_after", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.sni", "target_field": "ssl.server_name", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.issuerdn", "target_field": "ssl.certificate.issuer", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.session_resumed", "target_field": "ssl.session_resumed", "ignore_missing": true } }, + { "set": { "field": "dataset", "value": "ssl" } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.subject", "target_field": "ssl.certificate.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.serial", "target_field": "ssl.certificate.serial", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.fingerprint", "target_field": "ssl.certificate.fingerprint", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.version", "target_field": "ssl.version", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3.hash", "target_field": "hash.ja3", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3.hash.string", "target_field": "hash.ja3_string", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3s.hash", "target_field": "hash.ja3s", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3s.hash.string", "target_field": "hash.ja3s_string", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.notbefore", "target_field": "x509.certificate.not_valid_before", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.notafter", "target_field": "x509.certificate.not_valid_after", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.sni", "target_field": "ssl.server_name", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.issuerdn", "target_field": "ssl.certificate.issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.session_resumed", "target_field": "ssl.session_resumed", "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 09ce7fd9f..d0c07492e 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -23,7 +23,7 @@ { "rename": { "field": "message2.TTLs", "target_field": "dns.ttls", "ignore_missing": true } }, { "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } }, - { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "zeek.dns.tld" } }, + { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "zeek.common" } } ] } From bb6f3107bcf2edf221a9a0de69a4fa6ef91a09fe Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 10:29:40 -0500 Subject: [PATCH 08/24] [fix] idstools can run on an import node as well --- salt/common/tools/sbin/so-rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index 154ecf461..a14ee0708 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -46,7 +46,7 @@ def check_apply(args: dict): def find_minion_pillar(minion_id: str) -> str: if minion_id == None: - regex = '^.*_(manager|standalone)\.sls$' + regex = '^.*_(manager|standalone|import)\.sls$' else: regex = f'^{minion_id}\.sls$' From f7bef9200ba8d12687394d360a434f1f1512dc08 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 10:38:53 -0500 Subject: [PATCH 09/24] [fix] Only look for manager-type pillars * SID disabling is only managed globally for now, so don't give the option to edit a different pillar --- salt/common/tools/sbin/so-rules | 35 ++++++++++----------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index a14ee0708..4a7429f30 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -44,11 +44,8 @@ def check_apply(args: dict): return 0 -def find_minion_pillar(minion_id: str) -> str: - if minion_id == None: - regex = '^.*_(manager|standalone|import)\.sls$' - else: - regex = f'^{minion_id}\.sls$' +def find_minion_pillar() -> str: + regex = '^.*_(manager|standalone|import|eval)\.sls$' result = [] for root, _, files in os.walk(minion_pillar_dir): @@ -57,22 +54,15 @@ def find_minion_pillar(minion_id: str) -> str: result.append(os.path.join(root, f_minion_id)) if len(result) == 0: - if minion_id == None: - print_err('Could not find minion pillar with minion id matching *_manager or *_standalone') - sys.exit(3) - else: - print_err(f'Could not find minion pillar for minion id: {minion_id}') - sys.exit(3) + print_err('Could not find manager-type pillar (eval, standalone, manager, import.)') + print_err('Are you running this script on the manager?') + sys.exit(3) elif len(result) > 1: res_str = ', '.join(f'\"{result}\"') - if minion_id == None: - print_err('(This should not happen, the system is in an error state if you see this message.)') - print_err('More than one manager or standalone pillar exists, minion id\'s listed below:') - print_err(f' {res_str}') - sys.exit(3) - else: - print_err(f'Multiple minion pillars matched the minion id {minion_id}. Are you sure this is a complete minion id?') - sys.exit(3) + print_err('(This should not happen, the system is in an error state if you see this message.)') + print_err('More than one manager-type pillar exists, minion id\'s listed below:') + print_err(f' {res_str}') + sys.exit(3) else: return result[0] @@ -349,10 +339,6 @@ def main(): const=True, required=False, help="After updating rule configuration, apply the idstools state.") - main_parser.add_argument('--minion', - dest='minion_id', - required=False, - help='Defaults to manager (i.e. action applied to entire grid).') subcommand_desc = textwrap.dedent( """\ @@ -427,8 +413,7 @@ def main(): if not hasattr(args, 'remove'): args.remove = False - args.pillar = find_minion_pillar(args.minion_id) - + args.pillar = find_minion_pillar() if hasattr(args, 'func'): exit_code = args.func(args) From bef3a6921c462605877f87cc3df2ff3c618e1502 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 11:12:02 -0500 Subject: [PATCH 10/24] [fix] SID wildcards are not parsed by idstools, remove --- salt/common/tools/sbin/so-rules | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index 4a7429f30..9553b547c 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -97,22 +97,20 @@ def write_pillar(pillar: str, content: dict): sys.exit(3) -def check_sid_pattern(sid_pattern: str, sid_only: bool = False): +def check_sid_pattern(sid_pattern: str): message = f'SID {sid_pattern} is not valid, did you forget the \"re:\" prefix for a regex pattern?' - if sid_pattern.startswith('re:') and not sid_only: + if sid_pattern.startswith('re:'): r_string = sid_pattern[3:] if not valid_regex(r_string): print_err('Invalid regex pattern.') return False else: return True - elif sid_pattern == '*': - return True else: sid: int try: - sid = int(sid_pattern.replace('*', '')) + sid = int(sid_pattern) except: print_err(message) return False @@ -350,7 +348,7 @@ def main(): subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command') - sid_or_regex_help = 'A valid SID with optional wildcard (ex: "4321" or "432*") or a regular expression pattern (ex: "re:heartbleed|spectre")' + sid_or_regex_help = 'A valid SID (ex: "4321") or regular expression pattern (ex: "re:heartbleed|spectre")' # Disabled actions disabled = subparsers.add_parser('disabled') From 4bcb7403a92280988add23de79d979759def119e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 11:27:03 -0500 Subject: [PATCH 11/24] Add apply option to end of command --- salt/common/tools/sbin/so-rules | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index 9553b547c..50eb66d37 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -331,12 +331,10 @@ def main(): print_err('You must run this script as root') sys.exit(1) + apply_help='After updating rule configuration, apply the idstools state.' + main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) - main_parser.add_argument('--apply', - action='store_const', - const=True, - required=False, - help="After updating rule configuration, apply the idstools state.") + main_parser.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) subcommand_desc = textwrap.dedent( """\ @@ -357,10 +355,12 @@ def main(): disabled_add = disabled_sub.add_parser('add') disabled_add.set_defaults(func=add_rem_disabled) disabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) + disabled_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) disabled_rem = disabled_sub.add_parser('remove') disabled_rem.set_defaults(func=add_rem_disabled, remove=True) disabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) + disabled_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) disabled_list = disabled_sub.add_parser('list') disabled_list.set_defaults(func=list_disabled_rules) @@ -373,10 +373,12 @@ def main(): enabled_add = enabled_sub.add_parser('add') enabled_add.set_defaults(func=add_rem_enabled) enabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) + enabled_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) enabled_rem = enabled_sub.add_parser('remove') enabled_rem.set_defaults(func=add_rem_enabled, remove=True) enabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) + enabled_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) enabled_list = enabled_sub.add_parser('list') enabled_list.set_defaults(func=list_enabled_rules) @@ -394,12 +396,14 @@ def main(): modify_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) modify_add.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help) modify_add.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help) + modify_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) modify_rem = modify_sub.add_parser('remove') modify_rem.set_defaults(func=add_rem_modify, remove=True) modify_rem.add_argument('sid_pattern', metavar='SID', help=sid_or_regex_help) modify_rem.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help) modify_rem.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help) + modify_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) modify_list = modify_sub.add_parser('list') modify_list.set_defaults(func=list_modified_rules) From e65c9e5c7c0ac03ea4b1d6be9dd1d9f1eb6efdb8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 11:29:30 -0500 Subject: [PATCH 12/24] Don't expect apply arg at beginning of command --- salt/common/tools/sbin/so-rules | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index 50eb66d37..b0ae6969a 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -334,7 +334,6 @@ def main(): apply_help='After updating rule configuration, apply the idstools state.' main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) - main_parser.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) subcommand_desc = textwrap.dedent( """\ From bcce205430c526c19f138c3c3e3b24a3f19cd243 Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 22 Feb 2021 13:00:14 -0500 Subject: [PATCH 13/24] Improve support for Suricata metadata #2200 --- salt/elasticsearch/files/ingest/suricata.ftp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/elasticsearch/files/ingest/suricata.ftp b/salt/elasticsearch/files/ingest/suricata.ftp index 7d29fa708..492bd97e9 100644 --- a/salt/elasticsearch/files/ingest/suricata.ftp +++ b/salt/elasticsearch/files/ingest/suricata.ftp @@ -1,14 +1,14 @@ { "description" : "suricata.ftp", "processors" : [ - { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.reply", "target_field": "server.reply_message", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.completion_code", "target_field": "server.reply_code", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.reply_received", "target_field": "server.reply_received", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.command", "target_field": "ftp.command", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.command_data", "target_field": "ftp.command_data", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.dynamic_port", "target_field": "ftp.data_channel_destination.port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.reply", "target_field": "server.reply_message", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.completion_code", "target_field": "server.reply_code", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.reply_received", "target_field": "server.reply_received", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.command", "target_field": "ftp.command", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.command_data", "target_field": "ftp.argument", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.dynamic_port", "target_field": "ftp.data_channel_destination.port", "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] } From 71c7ffae3e7519df187d55b5bcc309a5a0271894 Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 22 Feb 2021 13:49:29 -0500 Subject: [PATCH 14/24] Improve support for Suricata metadata #2200 --- salt/elasticsearch/files/ingest/suricata.dns | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/suricata.dns b/salt/elasticsearch/files/ingest/suricata.dns index e0986c97c..85229ee92 100644 --- a/salt/elasticsearch/files/ingest/suricata.dns +++ b/salt/elasticsearch/files/ingest/suricata.dns @@ -3,10 +3,11 @@ "processors" : [ { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.type", "target_field": "dns.type", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.type", "target_field": "dns.query.type", "ignore_missing": true } }, { "rename": { "field": "message2.dns.tx_id", "target_field": "dns.id", "ignore_missing": true } }, { "rename": { "field": "message2.dns.version", "target_field": "dns.version", "ignore_missing": true } }, { "rename": { "field": "message2.dns.rrname", "target_field": "dns.query.name", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rrtype", "target_field": "dns.query.type_name", "ignore_missing": true } }, { "rename": { "field": "message2.dns.flags", "target_field": "dns.flags", "ignore_missing": true } }, { "rename": { "field": "message2.dns.qr", "target_field": "dns.qr", "ignore_missing": true } }, { "rename": { "field": "message2.dns.rd", "target_field": "dns.recursion.desired", "ignore_missing": true } }, From 6ed1cc3875a57b9d760cbbab853e02d03a1818d8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Feb 2021 14:02:37 -0500 Subject: [PATCH 15/24] Add Soup Functions --- salt/common/tools/sbin/soup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 09be9531c..dc26eec8b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -206,6 +206,7 @@ pillar_changes() { [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 [[ "$INSTALLEDVERSION" == 2.3.0 ]] || [[ "$INSTALLEDVERSION" == 2.3.1 ]] || [[ "$INSTALLEDVERSION" == 2.3.2 ]] || [[ "$INSTALLEDVERSION" == 2.3.10 ]] && 2.3.0_to_2.3.20 + [[ "$INSTALLEDVERSION" == 2.3.20 ]] || [[ "$INSTALLEDVERSION" == 2.3.21 ]] && 2.3.2X_to_2.3.30 } rc1_to_rc2() { @@ -330,6 +331,10 @@ rc3_to_2.3.0() { } +2.3.2X_to_2.3.30(){ + +} + space_check() { # Check to see if there is enough space CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') From 8fc82fa3ef0846523cd97ebc40be69d71be2dab8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 14:27:22 -0500 Subject: [PATCH 16/24] Fix minion pillar directory --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0c322742d..106012664 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -333,7 +333,7 @@ fun_2.3.0_to_2.3.20(){ fun_2.3.20_to_2.3.30() { # Replace any curly brace scalars with the same scalar in single quotes - readarray -t minion_pillars <<< "$(find /opt/so/saltstack/local/minions -type f -name '*.sls')" + readarray -t minion_pillars <<< "$(find /opt/so/saltstack/local/pillar/minions -type f -name '*.sls')" for pillar in "${minion_pillars[@]}"; do sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar" done From cdf766eeaef69e6c7e897182d9ae0295f88cfe04 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 14:30:26 -0500 Subject: [PATCH 17/24] explicitely -> explicitly --- salt/common/tools/sbin/so-rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rules index b0ae6969a..1ca5e1213 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rules @@ -257,7 +257,7 @@ def list_enabled_rules(args: dict): enabled = pillar_dict.get('idstools', {}).get('sids', {}).get('enabled') if enabled is None: - print('No rules explicitely enabled.') + print('No rules explicitly enabled.') return 0 else: print('Enabled rules:') From fd33a6cebe1a543bafca4e9e0f34e40e805d71b2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 15:32:18 -0500 Subject: [PATCH 18/24] Rename script, prompt user to apply if they didn't pass --apply --- salt/common/tools/sbin/{so-rules => so-rule} | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) rename salt/common/tools/sbin/{so-rules => so-rule} (97%) diff --git a/salt/common/tools/sbin/so-rules b/salt/common/tools/sbin/so-rule similarity index 97% rename from salt/common/tools/sbin/so-rules rename to salt/common/tools/sbin/so-rule index 1ca5e1213..173b37074 100644 --- a/salt/common/tools/sbin/so-rules +++ b/salt/common/tools/sbin/so-rule @@ -37,11 +37,20 @@ def print_err(string: str): def check_apply(args: dict): + cmd_arr = ['salt-call', 'state.apply', 'idstools', 'queue=True'] + if args.apply: print('Applying idstools state...') - return subprocess.run(['salt-call', 'state.apply', 'idstools', 'queue=True']) + return subprocess.run(cmd_arr) else: - return 0 + message = 'Would you like to apply your changes now? (y/N) ' + answer = input(message) + while answer.lower() not in [ 'y', 'n', '' ]: + answer = input(message) + if answer.lower() in [ 'n', '' ]: + return subprocess.run(cmd_arr) + else: + return 0 def find_minion_pillar() -> str: From e9b85337ff5d97ee8e37a9799aed00deb64e4247 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 15:41:09 -0500 Subject: [PATCH 19/24] [fix] Only prompt if entry doesn't exist, deep compare arrays --- salt/common/tools/sbin/so-rule | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index 173b37074..8507fba96 100644 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -36,19 +36,22 @@ def print_err(string: str): print(string, file=sys.stderr) -def check_apply(args: dict): +def check_apply(args: dict, prompt: bool = True): cmd_arr = ['salt-call', 'state.apply', 'idstools', 'queue=True'] if args.apply: print('Applying idstools state...') return subprocess.run(cmd_arr) else: - message = 'Would you like to apply your changes now? (y/N) ' - answer = input(message) - while answer.lower() not in [ 'y', 'n', '' ]: + if prompt: + message = 'Would you like to apply your changes now? (y/N) ' answer = input(message) - if answer.lower() in [ 'n', '' ]: - return subprocess.run(cmd_arr) + while answer.lower() not in [ 'y', 'n', '' ]: + answer = input(message) + if answer.lower() in [ 'n', '' ]: + return subprocess.run(cmd_arr) + else: + return 0 else: return 0 @@ -181,7 +184,7 @@ def add_rem_disabled(args: dict): temp_pillar_dict = add_to_sids(pillar_dict, 'disabled', args.sid_pattern) if temp_pillar_dict['idstools']['sids']['disabled'] == pillar_dict['idstools']['sids']['disabled']: - salt_proc = check_apply(args) + salt_proc = check_apply(args, prompt=False) return salt_proc else: pillar_dict = temp_pillar_dict @@ -245,8 +248,8 @@ def add_rem_enabled(args: dict): else: temp_pillar_dict = add_to_sids(pillar_dict, 'enabled', args.sid_pattern) - if temp_pillar_dict == pillar_dict: - salt_proc = check_apply(args) + if temp_pillar_dict['idstools']['sids']['enabled'] == pillar_dict['idstools']['sids']['enabled']: + salt_proc = check_apply(args, prompt=False) return salt_proc else: pillar_dict = temp_pillar_dict @@ -296,8 +299,8 @@ def add_rem_modify(args: dict): else: temp_pillar_dict = add_to_sids(pillar_dict, 'modify', string_val) - if temp_pillar_dict == pillar_dict: - salt_proc = check_apply(args) + if temp_pillar_dict['idstools']['sids']['modify'] == pillar_dict['idstools']['sids']['modify']: + salt_proc = check_apply(args, prompt=False) return salt_proc else: pillar_dict = temp_pillar_dict From b00cc888018e02b9d9089cdb73783c096cc60215 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 15:43:56 -0500 Subject: [PATCH 20/24] [fix] Unreverse apply prompt actions --- salt/common/tools/sbin/so-rule | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index 8507fba96..46efca568 100644 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -49,9 +49,9 @@ def check_apply(args: dict, prompt: bool = True): while answer.lower() not in [ 'y', 'n', '' ]: answer = input(message) if answer.lower() in [ 'n', '' ]: - return subprocess.run(cmd_arr) - else: return 0 + else: + return subprocess.run(cmd_arr) else: return 0 From 3e3c923ab9424f55cea31275f9f8fd81c737ff76 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 15:44:29 -0500 Subject: [PATCH 21/24] Arrange missing pillar error message better --- salt/common/tools/sbin/so-rule | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index 46efca568..d365d0856 100644 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -66,12 +66,11 @@ def find_minion_pillar() -> str: result.append(os.path.join(root, f_minion_id)) if len(result) == 0: - print_err('Could not find manager-type pillar (eval, standalone, manager, import.)') - print_err('Are you running this script on the manager?') + print_err('Could not find manager-type pillar (eval, standalone, manager, import). Are you running this script on the manager?') sys.exit(3) elif len(result) > 1: res_str = ', '.join(f'\"{result}\"') - print_err('(This should not happen, the system is in an error state if you see this message.)') + print_err('(This should not happen, the system is in an error state if you see this message.)\n') print_err('More than one manager-type pillar exists, minion id\'s listed below:') print_err(f' {res_str}') sys.exit(3) From fb3af255d9666b4e1e87b134c6d918725b792544 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 22 Feb 2021 15:50:07 -0500 Subject: [PATCH 22/24] Add more info to apply messaging --- salt/common/tools/sbin/so-rule | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index d365d0856..9654456b4 100644 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -38,19 +38,20 @@ def print_err(string: str): def check_apply(args: dict, prompt: bool = True): cmd_arr = ['salt-call', 'state.apply', 'idstools', 'queue=True'] - + if args.apply: - print('Applying idstools state...') + print('Configuration updated. Applying idstools state...') return subprocess.run(cmd_arr) else: if prompt: - message = 'Would you like to apply your changes now? (y/N) ' + message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' answer = input(message) while answer.lower() not in [ 'y', 'n', '' ]: answer = input(message) if answer.lower() in [ 'n', '' ]: return 0 else: + print('Applying idstools state...') return subprocess.run(cmd_arr) else: return 0 From fad87a8789e8c2a0c8ad379a2c2d3e0011e92d18 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Feb 2021 08:51:44 -0500 Subject: [PATCH 23/24] Fix function name (.20 -> .2X) --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 22bfbcf39..31b1219f1 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -206,7 +206,7 @@ pillar_changes() { [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 - [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.20_to_2.3.30 + [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 } rc1_to_rc2() { From 122e34b69cd95783146bba9b078c6ce1ef822628 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 23 Feb 2021 10:06:24 -0500 Subject: [PATCH 24/24] Configure fleet result.log to rotate --- salt/fleet/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index f286af347..1bb4e73d6 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -126,6 +126,8 @@ so-fleet: - KOLIDE_OSQUERY_STATUS_LOG_FILE=/var/log/fleet/status.log - KOLIDE_OSQUERY_RESULT_LOG_FILE=/var/log/osquery/result.log - KOLIDE_SERVER_URL_PREFIX=/fleet + - KOLIDE_FILESYSTEM_ENABLE_LOG_ROTATION=true + - KOLIDE_FILESYSTEM_ENABLE_LOG_COMPRESSION=true - binds: - /etc/pki/fleet.key:/ssl/server.key:ro - /etc/pki/fleet.crt:/ssl/server.cert:ro