From 59852841ffe9628b3833002de7d2b67dda8bd8b0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 15 Oct 2021 13:29:50 -0400 Subject: [PATCH 01/12] Add keyword subfield for event.module --- .../templates/so/so-endgame-template.json.jinja | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja index 824558e8f..e39a2fcf9 100644 --- a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja @@ -719,7 +719,12 @@ }, "module": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "original": { "doc_values": false, From b5cb47e0663275dac79ceca221c94f4865a687c6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 20 Oct 2021 16:43:55 -0400 Subject: [PATCH 02/12] Fix sbin perms --- salt/common/tools/sbin/so-elasticsearch-roles-load | 0 salt/common/tools/sbin/so-import-evtx | 0 salt/common/tools/sbin/so-playbook-import | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 salt/common/tools/sbin/so-elasticsearch-roles-load mode change 100644 => 100755 salt/common/tools/sbin/so-import-evtx mode change 100644 => 100755 salt/common/tools/sbin/so-playbook-import diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/common/tools/sbin/so-elasticsearch-roles-load old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-playbook-import b/salt/common/tools/sbin/so-playbook-import old mode 100644 new mode 100755 From 0ed2ce0766decd07def16f38b1a2eb68564c8183 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 20 Oct 2021 16:44:09 -0400 Subject: [PATCH 03/12] Fix validation.sh tests --- tests/validation.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/validation.sh b/tests/validation.sh index d16c8bbb9..6ec2a5247 100644 --- a/tests/validation.sh +++ b/tests/validation.sh @@ -1,6 +1,6 @@ #!/bin/bash -. ../salt/common/tools/sbin/so-common +. "$(dirname "$0")"/../salt/common/tools/sbin/so-common script_ret=0 @@ -106,7 +106,7 @@ test_fun 1 valid_dns_list "192.168.9." sleep 0.15s -header "int (default min: 1, default max: 1000)" +header "int (default min: 1, default max: 1000000000)" test_fun 0 valid_int "24" @@ -114,9 +114,9 @@ test_fun 0 valid_int "1" test_fun 0 valid_int "2" "2" -test_fun 0 valid_int "1000" +test_fun 0 valid_int "1000000000" -test_fun 1 valid_int "10001" +test_fun 1 valid_int "1000000001" test_fun 1 valid_int "24" "" "20" From 387d4d6ad56e438b30d0dd9503d7c132d4c824b7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 20 Oct 2021 16:44:57 -0400 Subject: [PATCH 04/12] Add so-deny script + rewrite so-allow to match so-deny --- salt/common/tools/sbin/so-deny | 209 +++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100755 salt/common/tools/sbin/so-deny diff --git a/salt/common/tools/sbin/so-deny b/salt/common/tools/sbin/so-deny new file mode 100755 index 000000000..c36a9b9d6 --- /dev/null +++ b/salt/common/tools/sbin/so-deny @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ipaddress +import textwrap +import os +import subprocess +import sys +import argparse +import re +from lxml import etree as ET +from xml.dom import minidom + + +LOCAL_SALT_DIR='/opt/so/saltstack/local' +WAZUH_CONF='/nsm/wazuh/etc/ossec.conf' +VALID_ROLES = { + 'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' }, + 'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' }, + 'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' }, + 'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' }, + 'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' }, + 's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' }, + 'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' }, + 'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' }, + 'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' } +} + + +def validate_ip_cidr(ip_cidr: str) -> bool: + try: + ipaddress.ip_address(ip_cidr) + except ValueError: + try: + ipaddress.ip_network(ip_cidr) + except ValueError: + return False + return True + + +def role_prompt() -> str: + print() + print('Choose the role for the IP or Range you would like to deny') + print() + for role in VALID_ROLES: + print(f'[{role}] - {VALID_ROLES[role]["desc"]}') + print() + role = input('Please enter your selection: ') + if role in VALID_ROLES.keys(): + return VALID_ROLES[role]['role'] + else: + print(f'Invalid role \'{role}\', please try again.', file=sys.stderr) + sys.exit(1) + + +def ip_prompt() -> str: + ip = input('Enter a single ip address or range to deny (ex: 10.10.10.10 or 10.10.0.0/16): ') + if validate_ip_cidr(ip): + return ip + else: + print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr) + sys.exit(1) + + +def wazuh_enabled() -> bool: + for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'): + with open(file, 'r') as pillar: + if 'wazuh: 1' in pillar.read(): + return True + return False + + +def root_to_str(root: ET.ElementTree) -> str: + xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '') + xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str) + + # Remove specific substrings to better format comments on intial parse/write + xml_str = re.sub(r' -', '', xml_str) + xml_str = re.sub(r' -->', ' -->', xml_str) + + dom = minidom.parseString(xml_str) + return dom.toprettyxml(indent=" ") + + +def rem_wl(ip): + parser = ET.XMLParser(remove_blank_text=True) + with open(WAZUH_CONF, 'rb') as wazuh_conf: + tree = ET.parse(wazuh_conf, parser) + root = tree.getroot() + + global_elems = root.findall(f"global/white_list[. = '{ip}']/..") + if len(global_elems) > 0: + for g_elem in global_elems: + ge_index = list(root).index(g_elem) + if ge_index > 0 and root[list(root).index(g_elem) - 1].tag == ET.Comment: + root.remove(root[ge_index - 1]) + root.remove(g_elem) + + with open(WAZUH_CONF, 'w') as out: + out.write(root_to_str(root)) + + +def apply(role: str, ip: str) -> int: + firewall_cmd = ['so-firewall', 'excludehost', role, ip] + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True'] + restart_wazuh_cmd = ['so-wazuh-restart'] + print(f'Removing {ip} from the {role} role. This can take a few seconds...') + cmd = subprocess.run(firewall_cmd) + if cmd.returncode == 0: + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + else: + return cmd.returncode + if cmd.returncode == 0: + if wazuh_enabled and role=='analyst': + try: + rem_wl(ip) + print(f'Removed whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + except Exception as e: + print(f'Failed to remove whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + print(e) + return 1 + print('Restarting OSSEC Server...') + cmd = subprocess.run(restart_wazuh_cmd) + else: + return cmd.returncode + else: + print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr) + return cmd.returncode + if cmd.returncode != 0: + print('Failed to restart OSSEC server.') + return cmd.returncode + + +def main(): + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(f'''\ + additional information: + To use this script in interactive mode call it with no arguments + ''' + )) + + group = main_parser.add_argument_group(title='roles') + group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp") + group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp") + group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp") + group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp") + group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp") + group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp") + group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp") + group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp") + group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp") + + ip_g = main_parser.add_argument_group(title='allow') + ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip') + + args = main_parser.parse_args(sys.argv[1:]) + + if args.roles is None: + role = role_prompt() + ip = ip_prompt() + try: + return_code = apply(role, ip) + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + sys.exit(return_code) + elif args.roles is not None and args.ip is None: + main_parser.print_help() + else: + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + + sys.exit(return_code) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.exit(1) From 0beeeb94bfde67023f3c7310b33b8d6e5bfb435a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 21 Oct 2021 10:48:17 -0400 Subject: [PATCH 05/12] Actually add new so-allow script --- salt/common/tools/sbin/so-allow | 323 ++++++++++++++++++-------------- 1 file changed, 183 insertions(+), 140 deletions(-) diff --git a/salt/common/tools/sbin/so-allow b/salt/common/tools/sbin/so-allow index c3cdc0ea2..1d240d840 100755 --- a/salt/common/tools/sbin/so-allow +++ b/salt/common/tools/sbin/so-allow @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env python3 # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # @@ -15,152 +15,195 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -. /usr/sbin/so-common +import ipaddress +import textwrap +import os +import subprocess +import sys +import argparse +import re +from lxml import etree as ET +from xml.dom import minidom +from datetime import datetime as dt +from datetime import timezone as tz -local_salt_dir=/opt/so/saltstack/local - -SKIP=0 - -function usage { - -cat << EOF - -Usage: $0 [-abefhoprsw] [ -i IP ] - -This program allows you to add a firewall rule to allow connections from a new IP address or CIDR range. - -If you run this program with no arguments, it will present a menu for you to choose your options. - -If you want to automate and skip the menu, you can pass the desired options as command line arguments. - -EXAMPLES - -To add 10.1.2.3 to the analyst role: -so-allow -a -i 10.1.2.3 - -To add 10.1.2.0/24 to the osquery role: -so-allow -o -i 10.1.2.0/24 - -EOF +LOCAL_SALT_DIR='/opt/so/saltstack/local' +WAZUH_CONF='/nsm/wazuh/etc/ossec.conf' +VALID_ROLES = { + 'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' }, + 'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' }, + 'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' }, + 'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' }, + 'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' }, + 's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' }, + 'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' }, + 'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' }, + 'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' } } -while getopts "ahfesprbowi:" OPTION -do - case $OPTION in - h) - usage - exit 0 - ;; - a) - FULLROLE="analyst" - SKIP=1 - ;; - b) - FULLROLE="beats_endpoint" - SKIP=1 - ;; - e) - FULLROLE="elasticsearch_rest" - SKIP=1 - ;; - f) - FULLROLE="strelka_frontend" - SKIP=1 - ;; - i) IP=$OPTARG - ;; - o) - FULLROLE="osquery_endpoint" - SKIP=1 - ;; - w) - FULLROLE="wazuh_agent" - SKIP=1 - ;; - s) - FULLROLE="syslog" - SKIP=1 - ;; - p) - FULLROLE="wazuh_api" - SKIP=1 - ;; - r) - FULLROLE="wazuh_authd" - SKIP=1 - ;; - *) - usage - exit 0 - ;; - esac -done -if [ "$SKIP" -eq 0 ]; then +def validate_ip_cidr(ip_cidr: str) -> bool: + try: + ipaddress.ip_address(ip_cidr) + except ValueError: + try: + ipaddress.ip_network(ip_cidr) + except ValueError: + return False + return True - echo "This program allows you to add a firewall rule to allow connections from a new IP address." - echo "" - echo "Choose the role for the IP or Range you would like to add" - echo "" - echo "[a] - Analyst - ports 80/tcp and 443/tcp" - echo "[b] - Logstash Beat - port 5044/tcp" - echo "[e] - Elasticsearch REST API - port 9200/tcp" - echo "[f] - Strelka frontend - port 57314/tcp" - echo "[o] - Osquery endpoint - port 8090/tcp" - echo "[s] - Syslog device - 514/tcp/udp" - echo "[w] - Wazuh agent - port 1514/tcp/udp" - echo "[p] - Wazuh API - port 55000/tcp" - echo "[r] - Wazuh registration service - 1515/tcp" - echo "" - echo "Please enter your selection:" - read -r ROLE - echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):" - read -r IP - if [ "$ROLE" == "a" ]; then - FULLROLE=analyst - elif [ "$ROLE" == "b" ]; then - FULLROLE=beats_endpoint - elif [ "$ROLE" == "e" ]; then - FULLROLE=elasticsearch_rest - elif [ "$ROLE" == "f" ]; then - FULLROLE=strelka_frontend - elif [ "$ROLE" == "o" ]; then - FULLROLE=osquery_endpoint - elif [ "$ROLE" == "w" ]; then - FULLROLE=wazuh_agent - elif [ "$ROLE" == "s" ]; then - FULLROLE=syslog - elif [ "$ROLE" == "p" ]; then - FULLROLE=wazuh_api - elif [ "$ROLE" == "r" ]; then - FULLROLE=wazuh_authd - else - echo "I don't recognize that role" - exit 1 - fi +def role_prompt() -> str: + print() + print('Choose the role for the IP or Range you would like to allow') + print() + for role in VALID_ROLES: + print(f'[{role}] - {VALID_ROLES[role]["desc"]}') + print() + role = input('Please enter your selection: ') + if role in VALID_ROLES.keys(): + return VALID_ROLES[role]['role'] + else: + print(f'Invalid role \'{role}\', please try again.', file=sys.stderr) + sys.exit(1) + -fi +def ip_prompt() -> str: + ip = input('Enter a single ip address or range to allow (ex: 10.10.10.10 or 10.10.0.0/16): ') + if validate_ip_cidr(ip): + return ip + else: + print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr) + sys.exit(1) -echo "Adding $IP to the $FULLROLE role. This can take a few seconds" -/usr/sbin/so-firewall includehost $FULLROLE $IP -salt-call state.apply firewall queue=True -# Check if Wazuh enabled -if grep -q -R "wazuh: 1" $local_salt_dir/pillar/*; then - # If analyst, add to Wazuh AR whitelist - if [ "$FULLROLE" == "analyst" ]; then - WAZUH_MGR_CFG="/nsm/wazuh/etc/ossec.conf" - if ! grep -q "$IP" $WAZUH_MGR_CFG ; then - DATE=$(date) - sed -i 's/<\/ossec_config>//' $WAZUH_MGR_CFG - sed -i '/^$/N;/^\n$/D' $WAZUH_MGR_CFG - echo -e "\n \n $IP\n \n" >> $WAZUH_MGR_CFG - echo "Added whitelist entry for $IP in $WAZUH_MGR_CFG." - echo - echo "Restarting OSSEC Server..." - /usr/sbin/so-wazuh-restart - fi - fi -fi +def wazuh_enabled() -> bool: + for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'): + with open(file, 'r') as pillar: + if 'wazuh: 1' in pillar.read(): + return True + return False + + +def root_to_str(root: ET.ElementTree) -> str: + xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '') + xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str) + xml_str = re.sub(r' -', '', xml_str) + xml_str = re.sub(r' -->', ' -->', xml_str) + dom = minidom.parseString(xml_str) + return dom.toprettyxml(indent=" ") + + +def add_wl(ip): + parser = ET.XMLParser(remove_blank_text=True) + with open(WAZUH_CONF, 'rb') as wazuh_conf: + tree = ET.parse(wazuh_conf, parser) + root = tree.getroot() + + source_comment = ET.Comment(f'Address {ip} added by /usr/sbin/so-allow on {dt.utcnow().replace(tzinfo=tz.utc).strftime("%a %b %e %H:%M:%S %Z %Y")}') + new_global = ET.Element("global") + new_wl = ET.SubElement(new_global, 'white_list') + new_wl.text = ip + + root.append(source_comment) + root.append(new_global) + + with open(WAZUH_CONF, 'w') as add_out: + add_out.write(root_to_str(root)) + + +def apply(role: str, ip: str) -> int: + firewall_cmd = ['so-firewall', 'includehost', role, ip] + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True'] + restart_wazuh_cmd = ['so-wazuh-restart'] + print(f'Adding {ip} to the {role} role. This can take a few seconds...') + cmd = subprocess.run(firewall_cmd) + if cmd.returncode == 0: + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + else: + return cmd.returncode + if cmd.returncode == 0: + if wazuh_enabled and role=='analyst': + try: + add_wl(ip) + print(f'Added whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + except Exception as e: + print(f'Failed to add whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + print(e) + return 1 + print('Restarting OSSEC Server...') + cmd = subprocess.run(restart_wazuh_cmd) + else: + return cmd.returncode + else: + print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr) + return cmd.returncode + if cmd.returncode != 0: + print('Failed to restart OSSEC server.') + return cmd.returncode + + +def main(): + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(f'''\ + additional information: + To use this script in interactive mode call it with no arguments + ''' + )) + + group = main_parser.add_argument_group(title='roles') + group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp") + group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp") + group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp") + group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp") + group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp") + group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp") + group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp") + group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp") + group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp") + + ip_g = main_parser.add_argument_group(title='allow') + ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip') + + args = main_parser.parse_args(sys.argv[1:]) + + if args.roles is None: + role = role_prompt() + ip = ip_prompt() + try: + return_code = apply(role, ip) + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + sys.exit(return_code) + elif args.roles is not None and args.ip is None: + main_parser.print_help() + else: + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + + sys.exit(return_code) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.exit(1) + From 15fe7512b711c78abda442204b7ab48ff508e49b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 21 Oct 2021 10:49:41 -0400 Subject: [PATCH 06/12] Install lxml during setup and in common state --- salt/common/init.sls | 2 ++ setup/so-functions | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index f993534b3..05dd7023f 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -101,6 +101,7 @@ commonpkgs: - python3-m2crypto - python3-mysqldb - python3-packaging + - python3-lxml - git - vim @@ -143,6 +144,7 @@ commonpkgs: - python36-m2crypto - python36-mysql - python36-packaging + - python36-lxml - yum-utils - device-mapper-persistent-data - lvm2 diff --git a/setup/so-functions b/setup/so-functions index f4d08e9a9..58fbca562 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2164,6 +2164,7 @@ saltify() { python36-m2crypto\ python36-mysql\ python36-packaging\ + python36-lxml\ yum-utils\ device-mapper-persistent-data\ lvm2\ @@ -2250,7 +2251,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" >> "$setup_log" 2>&1 || exit 1 fi } From 77ee1db44ceb5a76f0c7c3487469c18bf86e61e5 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 21 Oct 2021 12:56:03 -0400 Subject: [PATCH 07/12] Add .keyword subfield for conflict fields --- .../so/so-endgame-template.json.jinja | 150 +++++++++++++++--- 1 file changed, 129 insertions(+), 21 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja index e39a2fcf9..6d2b89b27 100644 --- a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja @@ -26,23 +26,48 @@ "properties": { "ephemeral_id": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "id": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "version": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -597,7 +622,12 @@ "properties": { "version": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -683,18 +713,33 @@ }, "category": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "code": { "ignore_above": 1024, "type": "keyword" }, "created": { - "type": "date" + "type": "date", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "dataset": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "duration": { "type": "long" @@ -711,7 +756,12 @@ "type": "keyword" }, "ingested": { - "type": "date" + "type": "date", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "kind": { "ignore_above": 1024, @@ -734,7 +784,12 @@ }, "outcome": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "provider": { "ignore_above": 1024, @@ -761,11 +816,21 @@ }, "timezone": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "url": { "ignore_above": 1024, @@ -1011,7 +1076,12 @@ }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "os": { "properties": { @@ -1144,11 +1214,21 @@ }, "method": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "referrer": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -1192,7 +1272,12 @@ "properties": { "level": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "logger": { "ignore_above": 1024, @@ -2154,7 +2239,12 @@ }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "node": { "properties": { @@ -2170,7 +2260,12 @@ }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "version": { "ignore_above": 1024, @@ -2182,7 +2277,12 @@ "properties": { "address": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "as": { "properties": { @@ -2338,7 +2438,12 @@ }, "tags": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "threat": { "properties": { @@ -2689,6 +2794,9 @@ }, "original": { "fields": { + "keyword": { + "type": "keyword" + }, "text": { "norms": false, "type": "text" From f374dcbb587fdf34a1b2819a225c545235a7614c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 21 Oct 2021 13:54:06 -0400 Subject: [PATCH 08/12] Check for IP environment variable in so-allow and so-deny --- salt/common/tools/sbin/so-allow | 32 ++++++++++++++++++-------------- salt/common/tools/sbin/so-deny | 32 ++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/salt/common/tools/sbin/so-allow b/salt/common/tools/sbin/so-allow index 1d240d840..769dcc1e9 100755 --- a/salt/common/tools/sbin/so-allow +++ b/salt/common/tools/sbin/so-allow @@ -183,21 +183,25 @@ def main(): return_code = e.errno sys.exit(return_code) elif args.roles is not None and args.ip is None: - main_parser.print_help() - else: - if validate_ip_cidr(args.ip): - try: - for role in args.roles: - return_code = apply(role, args.ip) - if return_code > 0: - break - except Exception as e: - print(f'Unexpected exception occurred: {e}', file=sys.stderr) - return_code = e.errno + if os.environ.get('IP') is None: + main_parser.print_help() + sys.exit(1) else: - print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) - return_code = 1 - + args.ip = os.environ['IP'] + + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + sys.exit(return_code) diff --git a/salt/common/tools/sbin/so-deny b/salt/common/tools/sbin/so-deny index c36a9b9d6..c13ea3f32 100755 --- a/salt/common/tools/sbin/so-deny +++ b/salt/common/tools/sbin/so-deny @@ -184,21 +184,25 @@ def main(): return_code = e.errno sys.exit(return_code) elif args.roles is not None and args.ip is None: - main_parser.print_help() - else: - if validate_ip_cidr(args.ip): - try: - for role in args.roles: - return_code = apply(role, args.ip) - if return_code > 0: - break - except Exception as e: - print(f'Unexpected exception occurred: {e}', file=sys.stderr) - return_code = e.errno + if os.environ.get('IP') is None: + main_parser.print_help() + sys.exit(1) else: - print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) - return_code = 1 - + args.ip = os.environ['IP'] + + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + sys.exit(return_code) From 6e34905b4291edc7db3e329483695be60c4c818d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 Oct 2021 15:28:37 -0400 Subject: [PATCH 09/12] Escape single quotes and allow for any character in node description --- salt/sensoroni/files/sensoroni.json | 2 +- setup/so-functions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 1a6e6fc8c..743021a7d 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -17,7 +17,7 @@ "agent": { "nodeId": "{{ grains.host | lower }}", "role": "{{ grains.role }}", - "description": "{{ DESCRIPTION }}", + "description": {{ DESCRIPTION | tojson }}, "address": "{{ ADDRESS }}", "model": "{{ MODEL }}", "pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }}, diff --git a/setup/so-functions b/setup/so-functions index 58fbca562..62d458911 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1509,7 +1509,7 @@ host_pillar() { " mainint: '$MNIC'"\ "sensoroni:"\ " node_address: '$MAINIP'"\ - " node_description: '$NODE_DESCRIPTION'"\ + " node_description: '${NODE_DESCRIPTION//\'/''}'"\ "" > "$pillar_file" } From 7fa43a276a1b86772f13b0a21c19343ae1910372 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Oct 2021 13:15:20 -0400 Subject: [PATCH 10/12] Rename default headers and host for HTTP input --- salt/logstash/pipelines/config/so/0011_input_endgame.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/logstash/pipelines/config/so/0011_input_endgame.conf b/salt/logstash/pipelines/config/so/0011_input_endgame.conf index b87d8e9b2..375585957 100644 --- a/salt/logstash/pipelines/config/so/0011_input_endgame.conf +++ b/salt/logstash/pipelines/config/so/0011_input_endgame.conf @@ -3,6 +3,8 @@ input { id => "endgame_data" port => 3765 codec => es_bulk + request_headers_target_field => client_headers + remote_host_target_field => client_host ssl => true ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] ssl_certificate => "/usr/share/logstash/filebeat.crt" From 3be0d05eeab7b6cf714fbabf58c34f2ab1a6c00e Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Oct 2021 13:16:30 -0400 Subject: [PATCH 11/12] Update field removal based on HTTP input changes --- .../logstash/pipelines/config/so/9900_output_endgame.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja b/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja index f23913637..b5920fe40 100644 --- a/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja +++ b/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja @@ -8,7 +8,7 @@ filter { if [event][module] =~ "endgame" { mutate { - remove_field => ["headers", "host"] + remove_field => ["client_headers", "client_host"] } } } From d5f42e0d7c8eac51ac4789dfcc5a9da9add6a3ef Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 25 Oct 2021 15:06:42 -0400 Subject: [PATCH 12/12] Update whiptail links to use latest docs --- setup/so-whiptail | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index ed4067da1..e74529438 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -285,7 +285,7 @@ whiptail_storage_requirements() { You need ${needed_val} to meet minimum requirements. - Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. + Visit https://docs.securityonion.net/en/latest/hardware.html for more information. Select YES to continue anyway, or select NO to cancel. EOM @@ -1774,7 +1774,7 @@ whiptail_storage_requirements() { You need ${needed_val} to meet minimum requirements. - Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. + Visit https://docs.securityonion.net/en/latest/hardware.html for more information. Press YES to continue anyway, or press NO to cancel. EOM