diff --git a/HOTFIX b/HOTFIX index e69de29bb..8b1378917 100644 --- a/HOTFIX +++ b/HOTFIX @@ -0,0 +1 @@ + diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 10fab2ed1..55b2070ce 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -13,3 +13,4 @@ logstash: - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja + - so/9800_output_logscan.conf.jinja diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 665fdbe3d..00b3e6007 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -45,7 +45,8 @@ 'schedule', 'soctopus', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'learn' ], 'so-heavynode': [ 'ca', @@ -108,7 +109,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'learn' ], 'so-manager': [ 'salt.master', @@ -127,7 +129,8 @@ 'utility', 'schedule', 'soctopus', - 'docker_clean' + 'docker_clean', + 'learn' ], 'so-managersearch': [ 'salt.master', @@ -146,7 +149,8 @@ 'utility', 'schedule', 'soctopus', - 'docker_clean' + 'docker_clean', + 'learn' ], 'so-node': [ 'ca', @@ -178,7 +182,8 @@ 'schedule', 'soctopus', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'learn' ], 'so-sensor': [ 'ca', @@ -237,7 +242,7 @@ {% do allowed_states.append('kibana') %} {% endif %} - {% if CURATOR and grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode'] %} + {% if grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-manager'] %} {% do allowed_states.append('curator') %} {% endif %} @@ -296,4 +301,4 @@ {% endif %} {# all nodes can always run salt.minion state #} -{% do allowed_states.append('salt.minion') %} \ No newline at end of file +{% do allowed_states.append('salt.minion') %} diff --git a/salt/common/files/log-rotate.conf b/salt/common/files/log-rotate.conf index 061b76271..35c6fd724 100644 --- a/salt/common/files/log-rotate.conf +++ b/salt/common/files/log-rotate.conf @@ -22,6 +22,7 @@ /opt/so/log/salt/so-salt-minion-check /opt/so/log/salt/minion /opt/so/log/salt/master +/opt/so/log/logscan/*.log { {{ logrotate_conf | indent(width=4) }} } diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete new file mode 100644 index 000000000..f3338de84 --- /dev/null +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -0,0 +1,17 @@ +# this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 + +folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')}}@localhost:3000/api/folders | jq -r '.[] | @base64') +delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") + +for row in $folders; do + title=$(echo ${row} | base64 --decode | jq -r '.title') + uid=$(echo ${row} | base64 --decode | jq -r '.uid') + + if [[ " ${delfolder[@]} " =~ " ${title} " ]]; then + curl -X DELETE http://admin:{{salt['pillar.get']('secrets:grafana_admin')}}@localhost:3000/api/folders/$uid + fi +done + +echo "so-grafana-dashboard-folder-delete has been run to delete default Grafana dashboard folders that existed prior to 2.3.70" > /opt/so/state/so-grafana-dashboard-folder-delete-complete + +exit 0 diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 9b6e2174a..cea0f6507 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -37,6 +37,7 @@ container_list() { "so-idstools" "so-kibana" "so-kratos" + "so-logscan" "so-nginx" "so-pcaptools" "so-soc" @@ -58,6 +59,7 @@ container_list() { "so-influxdb" "so-kibana" "so-kratos" + "so-logscan" "so-logstash" "so-mysql" "so-nginx" diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn new file mode 100644 index 000000000..f4a8ef90e --- /dev/null +++ b/salt/common/tools/sbin/so-learn @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from typing import List + +import signal +import sys +import os +import re +import subprocess +import argparse +import textwrap +import yaml +import multiprocessing + +minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' +so_status_conf = '/opt/so/conf/so-status/so-status.conf' +salt_proc: subprocess.CompletedProcess = None + +# Temp store of modules, will likely be broken out into salt +def get_learn_modules(): + return { + 'logscan': { 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } + } + + +def get_cpu_period(fraction: float): + multiplier = 10000 + + num_cores = multiprocessing.cpu_count() + if num_cores <= 2: + fraction = 1. + + num_used_cores = int(num_cores * fraction) + cpu_period = num_used_cores * multiplier + return cpu_period + + +def sigint_handler(*_): + print('Exiting gracefully on Ctrl-C') + if salt_proc is not None: salt_proc.send_signal(signal.SIGINT) + sys.exit(0) + + +def find_minion_pillar() -> str: + regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$' + + result = [] + for root, _, files in os.walk(minion_pillar_dir): + for f_minion_id in files: + if re.search(regex, f_minion_id): + result.append(os.path.join(root, f_minion_id)) + + if len(result) == 0: + print('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?', file=sys.stderr) + sys.exit(3) + elif len(result) > 1: + res_str = ', '.join(f'\"{result}\"') + print('(This should not happen, the system is in an error state if you see this message.)\n', file=sys.stderr) + print('More than one manager-type pillar exists, minion id\'s listed below:', file=sys.stderr) + print(f' {res_str}', file=sys.stderr) + sys.exit(3) + else: + return result[0] + + +def read_pillar(pillar: str): + try: + with open(pillar, 'r') as pillar_file: + loaded_yaml = yaml.safe_load(pillar_file.read()) + if loaded_yaml is None: + print(f'Could not parse {pillar}', file=sys.stderr) + sys.exit(3) + return loaded_yaml + except: + print(f'Could not open {pillar}', file=sys.stderr) + sys.exit(3) + + +def write_pillar(pillar: str, content: dict): + try: + with open(pillar, 'w') as pillar_file: + yaml.dump(content, pillar_file, default_flow_style=False) + except: + print(f'Could not open {pillar}', file=sys.stderr) + sys.exit(3) + + +def mod_so_status(action: str, item: str): + with open(so_status_conf, 'a+') as conf: + conf.seek(0) + containers = conf.readlines() + + if f'so-{item}\n' in containers: + if action == 'remove': containers.remove(f'so-{item}\n') + if action == 'add': pass + else: + if action == 'remove': pass + if action == 'add': containers.append(f'so-{item}\n') + + + [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines + + conf.seek(0) + conf.truncate(0) + conf.writelines(containers) + + +def create_pillar_if_not_exist(pillar:str, content: dict): + pillar_dict = content + + if pillar_dict.get('learn', {}).get('modules') is None: + pillar_dict['learn'] = {} + pillar_dict['learn']['modules'] = get_learn_modules() + content.update() + write_pillar(pillar, content) + + return content + + +def apply(module_list: List): + return_code = 0 + for module in module_list: + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] + print(f' Applying salt state for {module} module...') + salt_proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + if salt_proc.returncode != 0: + print(f' [ERROR] Failed to apply salt state for {module} module.') + return_code = salt_proc.returncode + return return_code + + +def check_apply(args: dict): + if args.apply: + print('Configuration updated. Applying changes:') + return apply(args.modules) + else: + message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' + answer = input(message) + while answer.lower() not in [ 'y', 'n', '' ]: + answer = input(message) + if answer.lower() in [ 'n', '' ]: + return 0 + else: + print('Applying changes:') + return apply(args.modules) + + +def enable_disable_modules(args, enable: bool): + pillar_modules = args.pillar_dict.get('learn', {}).get('modules') + pillar_mod_names = args.pillar_dict.get('learn', {}).get('modules').keys() + + action_str = 'add' if enable else 'remove' + + if 'all' in args.modules: + for module, details in pillar_modules.items(): + details['enabled'] = enable + mod_so_status(action_str, module) + args.pillar_dict.update() + write_pillar(args.pillar, args.pillar_dict) + else: + write_needed = False + for module in args.modules: + if module in pillar_mod_names: + if pillar_modules[module]['enabled'] == enable: + state_str = 'enabled' if enable else 'disabled' + print(f'{module} module already {state_str}.', file=sys.stderr) + else: + pillar_modules[module]['enabled'] = enable + mod_so_status(action_str, module) + write_needed = True + if write_needed: + args.pillar_dict.update() + write_pillar(args.pillar, args.pillar_dict) + + cmd_ret = check_apply(args) + return cmd_ret + + +def enable_modules(args): + enable_disable_modules(args, enable=True) + + +def disable_modules(args): + enable_disable_modules(args, enable=False) + + +def list_modules(*_): + print('Available ML modules:') + for module, details in get_learn_modules().items(): + print(f' - { module } : {details["description"]}') + return 0 + + +def main(): + beta_str = 'BETA - SUBJECT TO CHANGE\n' + + apply_help='After ACTION the chosen modules, apply any necessary salt states.' + enable_apply_help = apply_help.replace('ACTION', 'enabling') + disable_apply_help = apply_help.replace('ACTION', 'disabling') + + signal.signal(signal.SIGINT, sigint_handler) + + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) + + subcommand_desc = textwrap.dedent( + """\ + enable Enable one or more ML modules. + disable Disable one or more ML modules. + list List all available ML modules. + """ + ) + + subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command') + + module_help_str = 'One or more ML modules, which can be listed using \'so-learn list\'. Use the keyword \'all\' to apply the action to all available modules.' + + enable = subparsers.add_parser('enable') + enable.set_defaults(func=enable_modules) + enable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str) + enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) + + disable = subparsers.add_parser('disable') + disable.set_defaults(func=disable_modules) + disable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str) + disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) + + list = subparsers.add_parser('list') + list.set_defaults(func=list_modules) + + args = main_parser.parse_args(sys.argv[1:]) + args.pillar = find_minion_pillar() + args.pillar_dict = create_pillar_if_not_exist(args.pillar, read_pillar(args.pillar)) + + if hasattr(args, 'func'): + exit_code = args.func(args) + else: + if args.command is None: + print(beta_str) + main_parser.print_help() + sys.exit(0) + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 3c1ce8fb9..e85474c67 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -31,7 +31,7 @@ if [[ $# -lt 1 ]]; then echo "Usage: $0 " echo echo "All PCAPs must be placed in the /opt/so/samples directory unless replaying" - echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP sampes" + echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP samples" echo "are located in the /opt/samples directory inside of the image." echo echo "Customer provided PCAP example:" diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 7dd7b82e7..9730b72fb 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -4,7 +4,7 @@ {%- if grains['role'] in ['so-node', 'so-heavynode'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%} -{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} +{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('manager:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('manager:es_port', '') -%} {%- endif -%} diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 956f60491..15ecb8cb1 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -1,6 +1,6 @@ {% if grains['role'] in ['so-node', 'so-heavynode'] %} {%- set elasticsearch = salt['pillar.get']('elasticsearch:mainip', '') -%} -{% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} +{% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} {%- set elasticsearch = salt['pillar.get']('manager:mainip', '') -%} {%- endif %} {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 48a10b4b8..35d5b2f1b 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -4,8 +4,9 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone'] %} +{% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone', 'so-manager'] %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} + {% from "curator/map.jinja" import CURATOROPTIONS with context %} # Curator # Create the group curatorgroup: @@ -118,8 +119,10 @@ so-curatordeletecron: - dayweek: '*' so-curator: - docker_container.running: + docker_container.{{ CURATOROPTIONS.status }}: + {% if CURATOROPTIONS.status == 'running' %} - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }} + - start: {{ CURATOROPTIONS.start }} - hostname: curator - name: so-curator - user: curator @@ -129,11 +132,31 @@ so-curator: - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - /opt/so/conf/curator/action/:/etc/curator/action:ro - /opt/so/log/curator:/var/log/curator:rw + - require: + - file: actionconfs + - file: curconf + - file: curlogdir + {% else %} + - force: True + {% endif %} append_so-curator_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-curator + - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf + + {% if not CURATOROPTIONS.start %} +so-curator_so-status.disabled: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-curator$ + {% else %} +delete_so-curator_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-curator$ + {% endif %} # Begin Curator Cron Jobs diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja new file mode 100644 index 000000000..e67587cae --- /dev/null +++ b/salt/curator/map.jinja @@ -0,0 +1,12 @@ +{% set CURATOROPTIONS = {} %} +{% set ENABLED = salt['pillar.get']('curator:enabled', True) %} +{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} + +# don't start the docker container if searchnode and true clustering is enabled or curator disabled via pillar or true cluster not enabled and manager +{% if not ENABLED or (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} + {% do CURATOROPTIONS.update({'start': False}) %} + {% do CURATOROPTIONS.update({'status': 'absent'}) %} +{% else %} + {% do CURATOROPTIONS.update({'start': True}) %} + {% do CURATOROPTIONS.update({'status': 'running'}) %} +{% endif %} diff --git a/salt/elasticsearch/files/ingest/logscan.alert b/salt/elasticsearch/files/ingest/logscan.alert new file mode 100644 index 000000000..7473060a7 --- /dev/null +++ b/salt/elasticsearch/files/ingest/logscan.alert @@ -0,0 +1,29 @@ +{ + "description": "logscan", + "processors": [ + { "set": { "field": "event.severity", "value": 2 } }, + { "json": { "field": "message", "add_to_root": true, "ignore_failure": true } }, + { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, + { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "remove": { "field": "start_time", "ignore_missing": true } }, + { "remove": { "field": "end_time", "ignore_missing": true } }, + { "rename": { "field": "source_ip", "target_field": "source.ip", "ignore_missing": true } }, + { "rename": { "field": "top_source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, + { "append": { "if": "ctx.source != null", "field": "logscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, + { "set": { "if": "ctx.model == 'k1'", "field": "rule.name", "value": "LOGSCAN K1 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k1'", "field": "rule.description", "value": "High number of logins from single IP in 1 minute window" } }, + { "set": { "if": "ctx.model == 'k5'", "field": "rule.name", "value": "LOGSCAN K5 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k5'", "field": "rule.description", "value": "High ratio of login failures from single IP in 5 minute window" } }, + { "set": { "if": "ctx.model == 'k60'", "field": "rule.name", "value": "LOGSCAN K60 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k60'", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "rename": { "field": "model", "target_field": "logscan.model" } }, + { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, + { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "ctx.logscan.attempts.succeeded.amount = ctx.logscan.attempts.total.amount - ctx.logscan.attempts.failed.amount" , "ignore_failure": true} }, + { "rename": { "field": "avg_failure_interval", "target_field": "logscan.attempts.failed.avg_interval", "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/suricata.fileinfo b/salt/elasticsearch/files/ingest/suricata.fileinfo index d5147fb40..fe9e4b109 100644 --- a/salt/elasticsearch/files/ingest/suricata.fileinfo +++ b/salt/elasticsearch/files/ingest/suricata.fileinfo @@ -13,6 +13,7 @@ { "rename": { "field": "message2.fileinfo.size", "target_field": "file.size", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.state", "target_field": "file.state", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.stored", "target_field": "file.saved", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.sha256", "target_field": "hash.sha256", "ignore_missing": true } }, { "set": { "if": "ctx.network?.protocol != null", "field": "file.source", "value": "{{network.protocol}}" } }, { "pipeline": { "name": "common" } } ] diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 8afac271c..777bf3f53 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -313,6 +313,10 @@ "type":"object", "dynamic": true }, + "logscan": { + "type": "object", + "dynamic": true + }, "manager":{ "type":"object", "dynamic": true diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index f904ccfa6..0c27e3c1b 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -112,6 +112,21 @@ filebeat.inputs: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true +{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %} +- type: log + paths: + - /logs/logscan/alerts.log + fields: + module: logscan + dataset: alert + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + clean_removed: true + close_removed: false +{%- endif %} + {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} {%- if ZEEKVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} @@ -294,6 +309,9 @@ output.elasticsearch: - index: "so-strelka" when.contains: module: "strelka" + - index: "so-logscan" + when.contains: + module: "logscan" setup.template.enabled: false {%- else %} diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index dbb659217..1c1bdfb2e 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -44,6 +44,12 @@ grafanadashdir: - group: 939 - makedirs: True +{% for type in ['eval','manager','managersearch','search_nodes','sensor_nodes','standalone'] %} +remove_dashboard_dir_{{type}}: + file.absent: + - name: /opt/so/conf/grafana/grafana_dashboards/{{type}} +{% endfor %} + grafana-dashboard-config: file.managed: - name: /opt/so/conf/grafana/etc/dashboards/dashboard.yml @@ -82,6 +88,11 @@ grafana-config-files: - source: salt://grafana/etc/files - makedirs: True +so-grafana-dashboard-folder-delete: + cmd.run: + - name: /usr/sbin/so-grafana-dashboard-folder-delete + - unless: ls /opt/so/state/so-grafana-dashboard-folder-delete-complete + {% for dashboard in DASHBOARDS %} {{dashboard}}-dashboard: file.managed: diff --git a/salt/learn/files/logscan.conf b/salt/learn/files/logscan.conf new file mode 100644 index 000000000..d7aa30734 --- /dev/null +++ b/salt/learn/files/logscan.conf @@ -0,0 +1,7 @@ +[global] +ts_format = iso8601 +scan_interval = 30s +log_level = info + +[kratos] +log_path = kratos/kratos.log diff --git a/salt/learn/init.sls b/salt/learn/init.sls new file mode 100644 index 000000000..fb5b89802 --- /dev/null +++ b/salt/learn/init.sls @@ -0,0 +1,19 @@ +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} + +{% set module_dict = salt['pillar.get']('learn:modules', {} ) %} + +{% if module_dict.items()|length != 0 %} +include: +{% for module, _ in module_dict.items() %} + - 'learn.{{ module }}' +{% endfor %} +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls new file mode 100644 index 000000000..cc8bb2996 --- /dev/null +++ b/salt/learn/logscan.sls @@ -0,0 +1,56 @@ +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{% set MANAGER = salt['grains.get']('master') %} +{% set logscan_cpu_period = salt['pillar.get']('learn:modules:logscan:cpu_period', 20000) %} +{% set enabled = salt['pillar.get']('learn:modules:logscan:enabled', False) %} + +{% if enabled %} + {% set container_action = 'running' %} +{% else %} + {% set container_action = 'absent'%} +{% endif %} + + +logscan_data_dir: + file.directory: + - name: /nsm/logscan/data + - user: 939 + - group: 939 + - makedirs: True + +logscan_conf_dir: + file.directory: + - name: /opt/so/conf/logscan + - user: 939 + - group: 939 + - makedirs: True + +logscan_conf: + file.managed: + - name: /opt/so/conf/logscan/logscan.conf + - source: salt://learn/files/logscan.conf + - user: 939 + - group: 939 + - mode: 600 + +logscan_log_dir: + file.directory: + - name: /opt/so/log/logscan + - user: 939 + - group: 939 + +so-logscan: + docker_container.{{ container_action }}: + {% if container_action == 'running' %} + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logscan:{{ VERSION }} + - hostname: logscan + - name: so-logscan + - binds: + - /nsm/logscan/data:/logscan/data:rw + - /opt/so/conf/logscan/logscan.conf:/logscan/logscan.conf:ro + - /opt/so/log/logscan:/logscan/output:rw + - /opt/so/log:/logscan/logs:ro + - cpu_period: {{ logscan_cpu_period }} + {% else %} + - force: true + {% endif %} diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja new file mode 100644 index 000000000..86944d155 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -0,0 +1,27 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + +output { + if [module] =~ "logscan" { + elasticsearch { + id => "logscan_pipeline" + pipeline => "logscan.alert" + hosts => "{{ ES }}" + {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + {% endif %} + index => "so-logscan" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} diff --git a/salt/suricata/suricata_meta.yaml b/salt/suricata/suricata_meta.yaml index 90b220000..1c3855501 100644 --- a/salt/suricata/suricata_meta.yaml +++ b/salt/suricata/suricata_meta.yaml @@ -7,7 +7,7 @@ suricata: dir: /nsm/extracted #write-fileinfo: "yes" #force-filestore: "yes" - #stream-depth: 0 + stream-depth: 0 #max-open-files: 1000 #force-hash: [sha1, md5] xff: diff --git a/salt/top.sls b/salt/top.sls index 24be8283c..4f0792c0c 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -11,7 +11,6 @@ {% set FILEBEAT = salt['pillar.get']('filebeat:enabled', True) %} {% set KIBANA = salt['pillar.get']('kibana:enabled', True) %} {% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %} -{% set CURATOR = salt['pillar.get']('curator:enabled', True) %} {% set REDIS = salt['pillar.get']('redis:enabled', True) %} {% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %} {% import_yaml 'salt/minion.defaults.yaml' as saltversion %} @@ -127,9 +126,7 @@ base: {%- if FILEBEAT %} - filebeat {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if ELASTALERT %} - elastalert {%- endif %} @@ -156,6 +153,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_manager and G@saltversion:{{saltversion}}': - match: compound @@ -191,6 +189,7 @@ base: {%- if KIBANA %} - kibana {%- endif %} + - curator {%- if ELASTALERT %} - elastalert {%- endif %} @@ -218,6 +217,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_standalone and G@saltversion:{{saltversion}}': - match: compound @@ -265,9 +265,7 @@ base: {%- if FILEBEAT %} - filebeat {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if ELASTALERT %} - elastalert {%- endif %} @@ -292,6 +290,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -310,9 +309,7 @@ base: {%- if LOGSTASH %} - logstash {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if FILEBEAT %} - filebeat {%- endif %} @@ -354,9 +351,7 @@ base: {%- if REDIS %} - redis {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if KIBANA %} - kibana {%- endif %} @@ -366,7 +361,6 @@ base: {%- if FILEBEAT %} - filebeat {%- endif %} - - utility - schedule {%- if FLEETMANAGER or FLEETNODE %} @@ -388,6 +382,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_heavynode and G@saltversion:{{saltversion}}': - match: compound @@ -409,9 +404,7 @@ base: {%- if REDIS %} - redis {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if FILEBEAT %} - filebeat {%- endif %} @@ -478,3 +471,4 @@ base: - schedule - docker_clean - pipeline.load + - learn diff --git a/setup/automation/standalone-iso-logscan b/setup/automation/standalone-iso-logscan new file mode 100644 index 000000000..d83ad73db --- /dev/null +++ b/setup/automation/standalone-iso-logscan @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +LEARN_LOGSCAN_ENABLE=true +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/so-setup b/setup/so-setup index 68490657f..958d8aea1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -962,6 +962,12 @@ else set_progress_str 99 'Waiting for TheHive to start up' check_hive_init >> $setup_log 2>&1 fi + + if [[ -n $LEARN_LOGSCAN_ENABLE ]]; then + set_progress_str 99 'Enabling logscan' + so-learn enable logscan --apply >> $setup_log 2>&1 + fi + } | whiptail_gauge_post_setup "Running post-installation steps..." whiptail_setup_complete