diff --git a/salt/common/init.sls b/salt/common/init.sls index f993534b3..05dd7023f 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -101,6 +101,7 @@ commonpkgs: - python3-m2crypto - python3-mysqldb - python3-packaging + - python3-lxml - git - vim @@ -143,6 +144,7 @@ commonpkgs: - python36-m2crypto - python36-mysql - python36-packaging + - python36-lxml - yum-utils - device-mapper-persistent-data - lvm2 diff --git a/salt/common/tools/sbin/so-allow b/salt/common/tools/sbin/so-allow index c3cdc0ea2..769dcc1e9 100755 --- a/salt/common/tools/sbin/so-allow +++ b/salt/common/tools/sbin/so-allow @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env python3 # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # @@ -15,152 +15,199 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -. /usr/sbin/so-common +import ipaddress +import textwrap +import os +import subprocess +import sys +import argparse +import re +from lxml import etree as ET +from xml.dom import minidom +from datetime import datetime as dt +from datetime import timezone as tz -local_salt_dir=/opt/so/saltstack/local - -SKIP=0 - -function usage { - -cat << EOF - -Usage: $0 [-abefhoprsw] [ -i IP ] - -This program allows you to add a firewall rule to allow connections from a new IP address or CIDR range. - -If you run this program with no arguments, it will present a menu for you to choose your options. - -If you want to automate and skip the menu, you can pass the desired options as command line arguments. - -EXAMPLES - -To add 10.1.2.3 to the analyst role: -so-allow -a -i 10.1.2.3 - -To add 10.1.2.0/24 to the osquery role: -so-allow -o -i 10.1.2.0/24 - -EOF +LOCAL_SALT_DIR='/opt/so/saltstack/local' +WAZUH_CONF='/nsm/wazuh/etc/ossec.conf' +VALID_ROLES = { + 'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' }, + 'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' }, + 'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' }, + 'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' }, + 'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' }, + 's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' }, + 'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' }, + 'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' }, + 'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' } } -while getopts "ahfesprbowi:" OPTION -do - case $OPTION in - h) - usage - exit 0 - ;; - a) - FULLROLE="analyst" - SKIP=1 - ;; - b) - FULLROLE="beats_endpoint" - SKIP=1 - ;; - e) - FULLROLE="elasticsearch_rest" - SKIP=1 - ;; - f) - FULLROLE="strelka_frontend" - SKIP=1 - ;; - i) IP=$OPTARG - ;; - o) - FULLROLE="osquery_endpoint" - SKIP=1 - ;; - w) - FULLROLE="wazuh_agent" - SKIP=1 - ;; - s) - FULLROLE="syslog" - SKIP=1 - ;; - p) - FULLROLE="wazuh_api" - SKIP=1 - ;; - r) - FULLROLE="wazuh_authd" - SKIP=1 - ;; - *) - usage - exit 0 - ;; - esac -done -if [ "$SKIP" -eq 0 ]; then +def validate_ip_cidr(ip_cidr: str) -> bool: + try: + ipaddress.ip_address(ip_cidr) + except ValueError: + try: + ipaddress.ip_network(ip_cidr) + except ValueError: + return False + return True - echo "This program allows you to add a firewall rule to allow connections from a new IP address." - echo "" - echo "Choose the role for the IP or Range you would like to add" - echo "" - echo "[a] - Analyst - ports 80/tcp and 443/tcp" - echo "[b] - Logstash Beat - port 5044/tcp" - echo "[e] - Elasticsearch REST API - port 9200/tcp" - echo "[f] - Strelka frontend - port 57314/tcp" - echo "[o] - Osquery endpoint - port 8090/tcp" - echo "[s] - Syslog device - 514/tcp/udp" - echo "[w] - Wazuh agent - port 1514/tcp/udp" - echo "[p] - Wazuh API - port 55000/tcp" - echo "[r] - Wazuh registration service - 1515/tcp" - echo "" - echo "Please enter your selection:" - read -r ROLE - echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):" - read -r IP - if [ "$ROLE" == "a" ]; then - FULLROLE=analyst - elif [ "$ROLE" == "b" ]; then - FULLROLE=beats_endpoint - elif [ "$ROLE" == "e" ]; then - FULLROLE=elasticsearch_rest - elif [ "$ROLE" == "f" ]; then - FULLROLE=strelka_frontend - elif [ "$ROLE" == "o" ]; then - FULLROLE=osquery_endpoint - elif [ "$ROLE" == "w" ]; then - FULLROLE=wazuh_agent - elif [ "$ROLE" == "s" ]; then - FULLROLE=syslog - elif [ "$ROLE" == "p" ]; then - FULLROLE=wazuh_api - elif [ "$ROLE" == "r" ]; then - FULLROLE=wazuh_authd - else - echo "I don't recognize that role" - exit 1 - fi +def role_prompt() -> str: + print() + print('Choose the role for the IP or Range you would like to allow') + print() + for role in VALID_ROLES: + print(f'[{role}] - {VALID_ROLES[role]["desc"]}') + print() + role = input('Please enter your selection: ') + if role in VALID_ROLES.keys(): + return VALID_ROLES[role]['role'] + else: + print(f'Invalid role \'{role}\', please try again.', file=sys.stderr) + sys.exit(1) + -fi +def ip_prompt() -> str: + ip = input('Enter a single ip address or range to allow (ex: 10.10.10.10 or 10.10.0.0/16): ') + if validate_ip_cidr(ip): + return ip + else: + print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr) + sys.exit(1) -echo "Adding $IP to the $FULLROLE role. This can take a few seconds" -/usr/sbin/so-firewall includehost $FULLROLE $IP -salt-call state.apply firewall queue=True -# Check if Wazuh enabled -if grep -q -R "wazuh: 1" $local_salt_dir/pillar/*; then - # If analyst, add to Wazuh AR whitelist - if [ "$FULLROLE" == "analyst" ]; then - WAZUH_MGR_CFG="/nsm/wazuh/etc/ossec.conf" - if ! grep -q "$IP" $WAZUH_MGR_CFG ; then - DATE=$(date) - sed -i 's/<\/ossec_config>//' $WAZUH_MGR_CFG - sed -i '/^$/N;/^\n$/D' $WAZUH_MGR_CFG - echo -e "\n \n $IP\n \n" >> $WAZUH_MGR_CFG - echo "Added whitelist entry for $IP in $WAZUH_MGR_CFG." - echo - echo "Restarting OSSEC Server..." - /usr/sbin/so-wazuh-restart - fi - fi -fi +def wazuh_enabled() -> bool: + for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'): + with open(file, 'r') as pillar: + if 'wazuh: 1' in pillar.read(): + return True + return False + + +def root_to_str(root: ET.ElementTree) -> str: + xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '') + xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str) + xml_str = re.sub(r' -', '', xml_str) + xml_str = re.sub(r' -->', ' -->', xml_str) + dom = minidom.parseString(xml_str) + return dom.toprettyxml(indent=" ") + + +def add_wl(ip): + parser = ET.XMLParser(remove_blank_text=True) + with open(WAZUH_CONF, 'rb') as wazuh_conf: + tree = ET.parse(wazuh_conf, parser) + root = tree.getroot() + + source_comment = ET.Comment(f'Address {ip} added by /usr/sbin/so-allow on {dt.utcnow().replace(tzinfo=tz.utc).strftime("%a %b %e %H:%M:%S %Z %Y")}') + new_global = ET.Element("global") + new_wl = ET.SubElement(new_global, 'white_list') + new_wl.text = ip + + root.append(source_comment) + root.append(new_global) + + with open(WAZUH_CONF, 'w') as add_out: + add_out.write(root_to_str(root)) + + +def apply(role: str, ip: str) -> int: + firewall_cmd = ['so-firewall', 'includehost', role, ip] + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True'] + restart_wazuh_cmd = ['so-wazuh-restart'] + print(f'Adding {ip} to the {role} role. This can take a few seconds...') + cmd = subprocess.run(firewall_cmd) + if cmd.returncode == 0: + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + else: + return cmd.returncode + if cmd.returncode == 0: + if wazuh_enabled and role=='analyst': + try: + add_wl(ip) + print(f'Added whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + except Exception as e: + print(f'Failed to add whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + print(e) + return 1 + print('Restarting OSSEC Server...') + cmd = subprocess.run(restart_wazuh_cmd) + else: + return cmd.returncode + else: + print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr) + return cmd.returncode + if cmd.returncode != 0: + print('Failed to restart OSSEC server.') + return cmd.returncode + + +def main(): + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(f'''\ + additional information: + To use this script in interactive mode call it with no arguments + ''' + )) + + group = main_parser.add_argument_group(title='roles') + group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp") + group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp") + group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp") + group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp") + group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp") + group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp") + group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp") + group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp") + group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp") + + ip_g = main_parser.add_argument_group(title='allow') + ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip') + + args = main_parser.parse_args(sys.argv[1:]) + + if args.roles is None: + role = role_prompt() + ip = ip_prompt() + try: + return_code = apply(role, ip) + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + sys.exit(return_code) + elif args.roles is not None and args.ip is None: + if os.environ.get('IP') is None: + main_parser.print_help() + sys.exit(1) + else: + args.ip = os.environ['IP'] + + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + + sys.exit(return_code) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.exit(1) + diff --git a/salt/common/tools/sbin/so-deny b/salt/common/tools/sbin/so-deny new file mode 100755 index 000000000..c13ea3f32 --- /dev/null +++ b/salt/common/tools/sbin/so-deny @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ipaddress +import textwrap +import os +import subprocess +import sys +import argparse +import re +from lxml import etree as ET +from xml.dom import minidom + + +LOCAL_SALT_DIR='/opt/so/saltstack/local' +WAZUH_CONF='/nsm/wazuh/etc/ossec.conf' +VALID_ROLES = { + 'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' }, + 'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' }, + 'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' }, + 'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' }, + 'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' }, + 's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' }, + 'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' }, + 'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' }, + 'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' } +} + + +def validate_ip_cidr(ip_cidr: str) -> bool: + try: + ipaddress.ip_address(ip_cidr) + except ValueError: + try: + ipaddress.ip_network(ip_cidr) + except ValueError: + return False + return True + + +def role_prompt() -> str: + print() + print('Choose the role for the IP or Range you would like to deny') + print() + for role in VALID_ROLES: + print(f'[{role}] - {VALID_ROLES[role]["desc"]}') + print() + role = input('Please enter your selection: ') + if role in VALID_ROLES.keys(): + return VALID_ROLES[role]['role'] + else: + print(f'Invalid role \'{role}\', please try again.', file=sys.stderr) + sys.exit(1) + + +def ip_prompt() -> str: + ip = input('Enter a single ip address or range to deny (ex: 10.10.10.10 or 10.10.0.0/16): ') + if validate_ip_cidr(ip): + return ip + else: + print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr) + sys.exit(1) + + +def wazuh_enabled() -> bool: + for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'): + with open(file, 'r') as pillar: + if 'wazuh: 1' in pillar.read(): + return True + return False + + +def root_to_str(root: ET.ElementTree) -> str: + xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '') + xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str) + + # Remove specific substrings to better format comments on intial parse/write + xml_str = re.sub(r' -', '', xml_str) + xml_str = re.sub(r' -->', ' -->', xml_str) + + dom = minidom.parseString(xml_str) + return dom.toprettyxml(indent=" ") + + +def rem_wl(ip): + parser = ET.XMLParser(remove_blank_text=True) + with open(WAZUH_CONF, 'rb') as wazuh_conf: + tree = ET.parse(wazuh_conf, parser) + root = tree.getroot() + + global_elems = root.findall(f"global/white_list[. = '{ip}']/..") + if len(global_elems) > 0: + for g_elem in global_elems: + ge_index = list(root).index(g_elem) + if ge_index > 0 and root[list(root).index(g_elem) - 1].tag == ET.Comment: + root.remove(root[ge_index - 1]) + root.remove(g_elem) + + with open(WAZUH_CONF, 'w') as out: + out.write(root_to_str(root)) + + +def apply(role: str, ip: str) -> int: + firewall_cmd = ['so-firewall', 'excludehost', role, ip] + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True'] + restart_wazuh_cmd = ['so-wazuh-restart'] + print(f'Removing {ip} from the {role} role. This can take a few seconds...') + cmd = subprocess.run(firewall_cmd) + if cmd.returncode == 0: + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + else: + return cmd.returncode + if cmd.returncode == 0: + if wazuh_enabled and role=='analyst': + try: + rem_wl(ip) + print(f'Removed whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + except Exception as e: + print(f'Failed to remove whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr) + print(e) + return 1 + print('Restarting OSSEC Server...') + cmd = subprocess.run(restart_wazuh_cmd) + else: + return cmd.returncode + else: + print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr) + return cmd.returncode + if cmd.returncode != 0: + print('Failed to restart OSSEC server.') + return cmd.returncode + + +def main(): + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(f'''\ + additional information: + To use this script in interactive mode call it with no arguments + ''' + )) + + group = main_parser.add_argument_group(title='roles') + group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp") + group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp") + group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp") + group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp") + group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp") + group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp") + group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp") + group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp") + group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp") + + ip_g = main_parser.add_argument_group(title='allow') + ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip') + + args = main_parser.parse_args(sys.argv[1:]) + + if args.roles is None: + role = role_prompt() + ip = ip_prompt() + try: + return_code = apply(role, ip) + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + sys.exit(return_code) + elif args.roles is not None and args.ip is None: + if os.environ.get('IP') is None: + main_parser.print_help() + sys.exit(1) + else: + args.ip = os.environ['IP'] + + if validate_ip_cidr(args.ip): + try: + for role in args.roles: + return_code = apply(role, args.ip) + if return_code > 0: + break + except Exception as e: + print(f'Unexpected exception occurred: {e}', file=sys.stderr) + return_code = e.errno + else: + print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr) + return_code = 1 + + sys.exit(return_code) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.exit(1) diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/common/tools/sbin/so-elasticsearch-roles-load old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-playbook-import b/salt/common/tools/sbin/so-playbook-import old mode 100644 new mode 100755 diff --git a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja index 824558e8f..6d2b89b27 100644 --- a/salt/elasticsearch/templates/so/so-endgame-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-endgame-template.json.jinja @@ -26,23 +26,48 @@ "properties": { "ephemeral_id": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "id": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "version": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -597,7 +622,12 @@ "properties": { "version": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -683,18 +713,33 @@ }, "category": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "code": { "ignore_above": 1024, "type": "keyword" }, "created": { - "type": "date" + "type": "date", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "dataset": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "duration": { "type": "long" @@ -711,7 +756,12 @@ "type": "keyword" }, "ingested": { - "type": "date" + "type": "date", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "kind": { "ignore_above": 1024, @@ -719,7 +769,12 @@ }, "module": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "original": { "doc_values": false, @@ -729,7 +784,12 @@ }, "outcome": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "provider": { "ignore_above": 1024, @@ -756,11 +816,21 @@ }, "timezone": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "url": { "ignore_above": 1024, @@ -1006,7 +1076,12 @@ }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "os": { "properties": { @@ -1139,11 +1214,21 @@ }, "method": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "referrer": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } } } }, @@ -1187,7 +1272,12 @@ "properties": { "level": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "logger": { "ignore_above": 1024, @@ -2149,7 +2239,12 @@ }, "name": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "node": { "properties": { @@ -2165,7 +2260,12 @@ }, "type": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "version": { "ignore_above": 1024, @@ -2177,7 +2277,12 @@ "properties": { "address": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "as": { "properties": { @@ -2333,7 +2438,12 @@ }, "tags": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "threat": { "properties": { @@ -2684,6 +2794,9 @@ }, "original": { "fields": { + "keyword": { + "type": "keyword" + }, "text": { "norms": false, "type": "text" diff --git a/salt/logstash/pipelines/config/so/0011_input_endgame.conf b/salt/logstash/pipelines/config/so/0011_input_endgame.conf index b87d8e9b2..375585957 100644 --- a/salt/logstash/pipelines/config/so/0011_input_endgame.conf +++ b/salt/logstash/pipelines/config/so/0011_input_endgame.conf @@ -3,6 +3,8 @@ input { id => "endgame_data" port => 3765 codec => es_bulk + request_headers_target_field => client_headers + remote_host_target_field => client_host ssl => true ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] ssl_certificate => "/usr/share/logstash/filebeat.crt" diff --git a/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja b/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja index f23913637..b5920fe40 100644 --- a/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja +++ b/salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja @@ -8,7 +8,7 @@ filter { if [event][module] =~ "endgame" { mutate { - remove_field => ["headers", "host"] + remove_field => ["client_headers", "client_host"] } } } diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 1a6e6fc8c..743021a7d 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -17,7 +17,7 @@ "agent": { "nodeId": "{{ grains.host | lower }}", "role": "{{ grains.role }}", - "description": "{{ DESCRIPTION }}", + "description": {{ DESCRIPTION | tojson }}, "address": "{{ ADDRESS }}", "model": "{{ MODEL }}", "pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }}, diff --git a/setup/so-functions b/setup/so-functions index f4d08e9a9..62d458911 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1509,7 +1509,7 @@ host_pillar() { " mainint: '$MNIC'"\ "sensoroni:"\ " node_address: '$MAINIP'"\ - " node_description: '$NODE_DESCRIPTION'"\ + " node_description: '${NODE_DESCRIPTION//\'/''}'"\ "" > "$pillar_file" } @@ -2164,6 +2164,7 @@ saltify() { python36-m2crypto\ python36-mysql\ python36-packaging\ + python36-lxml\ yum-utils\ device-mapper-persistent-data\ lvm2\ @@ -2250,7 +2251,7 @@ saltify() { set_progress_str 8 'Installing salt-minion & python modules' retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" >> "$setup_log" 2>&1 || exit 1 fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index ed4067da1..e74529438 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -285,7 +285,7 @@ whiptail_storage_requirements() { You need ${needed_val} to meet minimum requirements. - Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. + Visit https://docs.securityonion.net/en/latest/hardware.html for more information. Select YES to continue anyway, or select NO to cancel. EOM @@ -1774,7 +1774,7 @@ whiptail_storage_requirements() { You need ${needed_val} to meet minimum requirements. - Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. + Visit https://docs.securityonion.net/en/latest/hardware.html for more information. Press YES to continue anyway, or press NO to cancel. EOM diff --git a/tests/validation.sh b/tests/validation.sh index d16c8bbb9..6ec2a5247 100644 --- a/tests/validation.sh +++ b/tests/validation.sh @@ -1,6 +1,6 @@ #!/bin/bash -. ../salt/common/tools/sbin/so-common +. "$(dirname "$0")"/../salt/common/tools/sbin/so-common script_ret=0 @@ -106,7 +106,7 @@ test_fun 1 valid_dns_list "192.168.9." sleep 0.15s -header "int (default min: 1, default max: 1000)" +header "int (default min: 1, default max: 1000000000)" test_fun 0 valid_int "24" @@ -114,9 +114,9 @@ test_fun 0 valid_int "1" test_fun 0 valid_int "2" "2" -test_fun 0 valid_int "1000" +test_fun 0 valid_int "1000000000" -test_fun 1 valid_int "10001" +test_fun 1 valid_int "1000000001" test_fun 1 valid_int "24" "" "20"