diff --git a/README.md b/README.md index fbf64b35e..a347d19ef 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,20 @@ ### Changes: -- Alpha 2 is here!! Check out the [Hybrid Hunter Quick Start Guide](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/Hybrid-Hunter-Quick-Start-Guide). -- Suricata 4.1.5 -- Bro/Zeek 2.6.4 -- TheHive 3.4.0 (ES to 6.8.3) -- NIDS and HIDS dashboard updates +- Alpha 2 is here! +- Suricata 4.1.5. +- Bro/Zeek 2.6.4. +- TheHive 3.4.0 (Includes ES 6.8.3 for TheHive only). +- Fixed Bro/Zeek packet loss calculation for Grafana. +- Updated to latest Sensoroni which includes websockets support for job status updates without having to refresh the page. +- NIDS and HIDS dashboard updates. - Playbook and ATT&CK Navigator features are now included. - Filebeat now logs to a file, instead of stdout. - Elastalert has been updated to use Python 3 and allow for use of custom alerters. -- Elasticsearch Ingest is now used to consume Zeek logs and Suricata alerts (instead of the traditional Logstash pipeline). - This reduces the memory footprint of Logstash dramatically! +- Moved Bro/Zeek log parsing from Logstash to Elasticsearch Ingest for higher performance and lower memory usage! - Several changes to the setup script have been made to improve stability of the setup process: - - Setup now modifies your hosts file so that the install works better in environments without DNS - - You are now prompted for setting a password for the socore user + - Setup now modifies your hosts file so that the install works better in environments without DNS. + - You are now prompted for setting a password for the socore user. - The install now forces a reboot at the end of the install. This fixes an issue with some of the Docker containers being in the wrong state from a manual reboot. Manual reboots are fine after the initial reboot. @@ -47,22 +48,9 @@ Distributed: ### Prerequisites -If you are running CentOS 7 there are a couple of prerequisites: - -``` -sudo yum -y install git bind-utils -sudo hostnamectl set-hostname YOURHOSTNAME -sudo reboot -``` - -If you are running CentOS 7 or Ubuntu 16.04 and don't have name resolution ensure your `/etc/hosts` file looks like this: - -``` -127.0.0.1 YOURHOSTNAME YOURHOSTNAME.localdomain localhost localhost.localdomain localhost4 localhost4.localdomain4 -::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 -``` -It is imperative that YOURHOSTNAME.localdomain is included in this hosts entry for the install to complete properly. +Install git if using a Centos 7 Minimal install: +```sudo yum -y install git``` ### Installation diff --git a/install_scripts/disable-checksum-offload.sh b/install_scripts/disable-checksum-offload.sh new file mode 100644 index 000000000..9cc0b5d5b --- /dev/null +++ b/install_scripts/disable-checksum-offload.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ "$NM_DISPATCHER_ACTION" == "pre-up" ]; then + if ["$DEVICE_IFACE" !== "$MAININT"]; then + for i in rx tx sg tso ufo gso gro lro; do + ethtool -K $DEVICE_IFACE $i off; + done + fi +fi \ No newline at end of file diff --git a/salt/bro/cron/packetloss.sh b/salt/bro/cron/packetloss.sh index 56dc7b771..744fd09f4 100644 --- a/salt/bro/cron/packetloss.sh +++ b/salt/bro/cron/packetloss.sh @@ -1,2 +1,2 @@ #!/bin/bash -/usr/bin/docker exec so-bro /opt/bro/bin/broctl netstats | awk -F '[ =]' '{RCVD += $5;DRP += $7;TTL += $9} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/bro/logs/packetloss.log +/usr/bin/docker exec so-bro /opt/bro/bin/broctl netstats | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/bro/logs/packetloss.log diff --git a/salt/bro/init.sls b/salt/bro/init.sls index 734255536..f406558be 100644 --- a/salt/bro/init.sls +++ b/salt/bro/init.sls @@ -125,13 +125,13 @@ localbrosync: so-broimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-bro:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-bro:HH1.1.1 so-bro: docker_container.running: - require: - so-broimage - - image: soshybridhunter/so-bro:HH1.1.0 + - image: soshybridhunter/so-bro:HH1.1.1 - privileged: True - binds: - /nsm/bro/logs:/nsm/bro/logs:rw diff --git a/salt/common/init.sls b/salt/common/init.sls index b61a0b7e9..f4a78a995 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -116,13 +116,13 @@ nginxtmp: # Start the core docker so-coreimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-core:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-core:HH1.1.1 so-core: docker_container.running: - require: - so-coreimage - - image: soshybridhunter/so-core:HH1.1.0 + - image: soshybridhunter/so-core:HH1.1.1 - hostname: so-core - user: socore - binds: diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index 3230e8edd..344ca4aed 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -113,6 +113,31 @@ http { } + location /playbook/ { + proxy_pass http://{{ masterip }}:3200/playbook/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + + location /navigator/ { + auth_basic "Security Onion"; + auth_basic_user_file /opt/so/conf/nginx/.htpasswd; + proxy_pass http://{{ masterip }}:4200/navigator/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + location /api/ { proxy_pass https://{{ masterip }}:8080/api/; proxy_read_timeout 90; @@ -173,6 +198,8 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; } diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index 7999a7027..dcedcafaf 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -113,6 +113,30 @@ http { } + location /playbook/ { + proxy_pass http://{{ masterip }}:3200/playbook/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /navigator/ { + auth_basic "Security Onion"; + auth_basic_user_file /opt/so/conf/nginx/.htpasswd; + proxy_pass http://{{ masterip }}:4200/navigator/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + location /api/ { proxy_pass https://{{ masterip }}:8080/api/; proxy_read_timeout 90; @@ -173,6 +197,8 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; } diff --git a/salt/common/tools/sbin/so-brologs b/salt/common/tools/sbin/so-bro-logs similarity index 100% rename from salt/common/tools/sbin/so-brologs rename to salt/common/tools/sbin/so-bro-logs diff --git a/salt/common/tools/sbin/so-getparsed b/salt/common/tools/sbin/so-get-parsed similarity index 100% rename from salt/common/tools/sbin/so-getparsed rename to salt/common/tools/sbin/so-get-parsed diff --git a/salt/common/tools/sbin/so-getunparsed b/salt/common/tools/sbin/so-get-unparsed similarity index 100% rename from salt/common/tools/sbin/so-getunparsed rename to salt/common/tools/sbin/so-get-unparsed diff --git a/salt/common/tools/sbin/so-listindex b/salt/common/tools/sbin/so-list-index similarity index 100% rename from salt/common/tools/sbin/so-listindex rename to salt/common/tools/sbin/so-list-index diff --git a/salt/elastalert/files/elastalert_config.yaml b/salt/elastalert/files/elastalert_config.yaml new file mode 100644 index 000000000..6a918093b --- /dev/null +++ b/salt/elastalert/files/elastalert_config.yaml @@ -0,0 +1,79 @@ +{% set esip = salt['pillar.get']('master:mainip', '') %} +{% set esport = salt['pillar.get']('master:es_port', '') %} +# This is the folder that contains the rule yaml files +# Any .yaml file will be loaded as a rule +rules_folder: /etc/elastalert/rules/ + +# Sets whether or not ElastAlert should recursively descend +# the rules directory - true or false +scan_subdirectories: true + +# How often ElastAlert will query Elasticsearch +# The unit can be anything from weeks to seconds +run_every: + minutes: 1 + +# ElastAlert will buffer results from the most recent +# period of time, in case some log sources are not in real time +buffer_time: + minutes: 1 + +# The maximum time between queries for ElastAlert to start at the most recently +# run query. When ElastAlert starts, for each rule, it will search elastalert_metadata +# for the most recently run query and start from that time, unless it is older than +# old_query_limit, in which case it will start from the present time. The default is one week. +old_query_limit: + minutes: 5 + +# The Elasticsearch hostname for metadata writeback +# Note that every rule can have its own Elasticsearch host +es_host: {{ esip }} + +# The Elasticsearch port +es_port: {{ esport }} + +# Sets timeout for connecting to and reading from es_host +es_conn_timeout: 60 + +# The maximum number of documents that will be downloaded from Elasticsearch in +# a single query. The default is 10,000, and if you expect to get near this number, +# consider using use_count_query for the rule. If this limit is reached, ElastAlert +# will scroll through pages the size of max_query_size until processing all results. +max_query_size: 5000 + +# The AWS region to use. Set this when using AWS-managed elasticsearch +#aws_region: us-east-1 + +# The AWS profile to use. Use this if you are using an aws-cli profile. +# See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html +# for details +#profile: test + +# Optional URL prefix for Elasticsearch +#es_url_prefix: elasticsearch + +# Connect with TLS to Elasticsearch +#use_ssl: True + +# Verify TLS certificates +#verify_certs: True + +# GET request with body is the default option for Elasticsearch. +# If it fails for some reason, you can pass 'GET', 'POST' or 'source'. +# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport +# for details +#es_send_get_body_as: GET + +# Option basic-auth username and password for Elasticsearch +#es_username: someusername +#es_password: somepassword + +# The index on es_host which is used for metadata storage +# This can be a unmapped index, but it is recommended that you run +# elastalert-create-index to set a mapping +writeback_index: elastalert_status + +# If an alert fails for some reason, ElastAlert will retry +# sending the alert until this time period has elapsed +alert_time_limit: + days: 2 diff --git a/salt/elastalert/files/modules/so/thehive.py b/salt/elastalert/files/modules/so/thehive.py new file mode 100644 index 000000000..42b6f9e1d --- /dev/null +++ b/salt/elastalert/files/modules/so/thehive.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import uuid +import re + +from elastalert.alerts import Alerter +from thehive4py.api import TheHiveApi +from thehive4py.models import Alert, AlertArtifact, CustomFieldHelper + +class TheHiveAlerter(Alerter): + """ + Use matched data to create alerts containing observables in an instance of TheHive + This is a modified version for use with Security Onion + """ + + required_options = set(['hive_connection', 'hive_alert_config']) + + def alert(self, matches): + + connection_details = self.rule['hive_connection'] + + api = TheHiveApi( + connection_details.get('hive_host'), + connection_details.get('hive_apikey', ''), + proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), + cert=connection_details.get('hive_verify', False)) + + for match in matches: + context = {'rule': self.rule, 'match': match} + + artifacts = [] + for mapping in self.rule.get('hive_observable_data_mapping', []): + for observable_type, match_data_key in mapping.items(): + try: + match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) + rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) + data_keys = match_data_keys + rule_data_keys + context_keys = list(context['match'].keys()) + list(context['rule'].keys()) + if all([True if k in context_keys else False for k in data_keys]): + artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) + except KeyError: + raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) + + alert_config = { + 'artifacts': artifacts, + 'sourceRef': str(uuid.uuid4())[0:6], + 'title': '{rule[index]}_{rule[name]}'.format(**context) + } + alert_config.update(self.rule.get('hive_alert_config', {})) + + for alert_config_field, alert_config_value in alert_config.items(): + if alert_config_field == 'customFields': + custom_fields = CustomFieldHelper() + for cf_key, cf_value in alert_config_value.items(): + try: + func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) + except AttributeError: + raise Exception('unsupported custom field type {}'.format(cf_value['type'])) + value = cf_value['value'].format(**context) + func(cf_key, value) + alert_config[alert_config_field] = custom_fields.build() + elif isinstance(alert_config_value, str): + alert_config[alert_config_field] = alert_config_value.format(**context) + elif isinstance(alert_config_value, (list, tuple)): + formatted_list = [] + for element in alert_config_value: + try: + formatted_list.append(element.format(**context)) + except (AttributeError, KeyError, IndexError): + formatted_list.append(element) + alert_config[alert_config_field] = formatted_list + + alert = Alert(**alert_config) + response = api.create_alert(alert) + + if response.status_code != 201: + raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) + + def get_info(self): + + return { + 'type': 'hivealerter', + 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') + } diff --git a/salt/elastalert/files/rules/so/nids2hive.yaml b/salt/elastalert/files/rules/so/nids2hive.yaml new file mode 100644 index 000000000..7d55b4675 --- /dev/null +++ b/salt/elastalert/files/rules/so/nids2hive.yaml @@ -0,0 +1,50 @@ +{% set es = salt['pillar.get']('static:masterip', '') %} +{% set hivehost = salt['pillar.get']('static:masterip', '') %} +{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +# hive.yaml +# Elastalert rule to forward IDS alerts from Security Onion to a specified TheHive instance. +# +es_host: {{es}} +es_port: 9200 +name: NIDS-Alert +type: frequency +index: "*:logstash-ids*" +num_events: 1 +timeframe: + minutes: 10 +buffer_time: + minutes: 10 +allow_buffer_time_overlap: true +query_key: alert +realert: + days: 1 + +filter: +- query: + query_string: + query: "event_type: ids AND NOT tags: _jsonparsefailure" + +alert: modules.so.thehive.TheHiveAlerter + +hive_connection: + hive_host: https://{{hivehost}}/thehive/ + hive_apikey: {{hivekey}} + +hive_proxies: + http: '' + https: '' + +hive_alert_config: + title: '{match[alert]}' + type: 'external' + source: 'SecurityOnion' + description: "`NIDS Dashboard:` \n\n \n\n `IPs: `{match[source_ip]}:{match[source_port]} --> {match[destination_ip]}:{match[destination_port]} \n\n `Signature:` {match[rule_signature]}" + severity: 2 + tags: ['elastalert', 'SecurityOnion', 'NIDS'] + tlp: 3 + status: 'New' + follow: True + +hive_observable_data_mapping: + - ip: '{match[source_ip]}' + - ip: '{match[destination_ip]}' diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 301aa9459..8e8b32ae6 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -64,38 +64,69 @@ elastarules: - group: 939 - makedirs: True -#elastaconfdir: -# file.directory: -# - name: /opt/so/conf/elastalert -# - user: 933 -# - group: 939 -# - makedirs: True +elastaconfdir: + file.directory: + - name: /opt/so/conf/elastalert + - user: 933 + - group: 939 + - makedirs: True -#elastaconf: -# file.managed: -# - name: /opt/so/conf/elastalert/config.yaml -# - source: salt://elastalert/files/config.yaml -# - user: 933 -# - group: 939 -# - template: jinja +elastasomodulesdir: + file.directory: + - name: /opt/so/conf/elastalert/modules/so + - user: 933 + - group: 939 + - makedirs: True + +elastacustmodulesdir: + file.directory: + - name: /opt/so/conf/elastalert/modules/custom + - user: 933 + - group: 939 + - makedirs: True + +elastasomodulesync: + file.recurse: + - name: /opt/so/conf/elastalert/modules/so + - source: salt://elastalert/files/modules/so + - user: 933 + - group: 939 + - makedirs: True + +elastarulesync: + file.recurse: + - name: /opt/so/rules/elastalert + - source: salt://elastalert/files/rules/so + - user: 933 + - group: 939 + - template: jinja + +elastaconf: + file.managed: + - name: /opt/so/conf/elastalert/elastalert_config.yaml + - source: salt://elastalert/files/elastalert_config.yaml + - user: 933 + - group: 939 + - template: jinja so-elastalertimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-elastalert:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-elastalert:HH1.1.1 so-elastalert: docker_container.running: - require: - so-elastalertimage - - image: soshybridhunter/so-elastalert:HH1.1.0 + - image: soshybridhunter/so-elastalert:HH1.1.1 - hostname: elastalert - name: so-elastalert - user: elastalert - detach: True - binds: -# - /opt/so/conf/elastalert/config.yaml:/etc/elastalert/conf/elastalert_config.yaml:ro - /opt/so/rules/elastalert:/etc/elastalert/rules/:ro - /opt/so/log/elastalert:/var/log/elastalert:rw + - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro + - /opt/so/conf/elastalert/elastalert_config.yaml:/etc/elastalert/conf/elastalert_config.yaml:ro - environment: - ELASTICSEARCH_HOST: {{ esip }} - ELASTICSEARCH_PORT: {{ esport }} diff --git a/salt/elasticsearch/files/ingest/bro_common b/salt/elasticsearch/files/ingest/bro_common new file mode 100644 index 000000000..98618ce56 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_common @@ -0,0 +1,9 @@ +{ + "description" : "bro_common", + "processors" : [ + { "rename": { "field": "@timestamp", "target_field": "timestamp", "ignore_missing": true } }, + { "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } }, + { "remove": { "field": "message2.ts", "ignore_failure": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_common_ssl b/salt/elasticsearch/files/ingest/bro_common_ssl new file mode 100644 index 000000000..faf1666ac --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_common_ssl @@ -0,0 +1,58 @@ +{ + "description" : "bro_common_ssl", + "processors" : [ + { + "kv": { + "field": "certificate_issuer", + "field_split": ",", + "value_split": "=", + "ignore_missing": true, + "ignore_failure": true, + "include_keys": [ "CN", "C", "O", "OU", "ST", "SN", "L", "DC", "GN", "pseudonym", "serialNumber", "title", "initials" ] + } + }, + { "rename":{ "field": "CN", "target_field": "issuer_common_name", "ignore_failure": true } }, + { "rename":{ "field": "C", "target_field": "issuer_country_code", "ignore_failure": true } }, + { "rename":{ "field": "O", "target_field": "issuer_organization", "ignore_failure": true } }, + { "rename":{ "field": "OU", "target_field": "issuer_organization_unit", "ignore_failure": true } }, + { "rename":{ "field": "ST", "target_field": "issuer_state", "ignore_failure": true } }, + { "rename":{ "field": "SN", "target_field": "issuer_surname", "ignore_failure": true } }, + { "rename":{ "field": "L", "target_field": "issuer_locality", "ignore_failure": true } }, + { "rename":{ "field": "DC", "target_field": "issuer_distinguised_name", "ignore_failure": true } }, + { "rename":{ "field": "GN", "target_field": "issuer_given_name", "ignore_failure": true } }, + { "rename":{ "field": "pseudonym", "target_field": "issuer_pseudonym", "ignore_failure": true } }, + { "rename":{ "field": "serialNumber", "target_field": "issuer_serial_number", "ignore_failure": true } }, + { "rename":{ "field": "title", "target_field": "issuer_title", "ignore_failure": true } }, + { "rename":{ "field": "initials", "target_field": "issuer_initials", "ignore_failure": true } }, + { + "kv": { + "field": "certificate_subject", + "field_split": ",", + "value_split": "=", + "ignore_missing": true, + "ignore_failure": true, + "include_keys": [ "CN", "C", "O", "OU", "ST", "SN", "L", "GN", "pseudonym", "serialNumber", "title", "initials" ] + } + }, + { "rename":{ "field": "CN", "target_field": "certificate_common_name", "ignore_failure": true } }, + { "rename":{ "field": "C", "target_field": "certificate_country_code", "ignore_failure": true } }, + { "rename":{ "field": "O", "target_field": "certificate_organization", "ignore_failure": true } }, + { "rename":{ "field": "OU", "target_field": "certificate_organization_unit","ignore_failure": true } }, + { "rename":{ "field": "ST", "target_field": "certificate_state", "ignore_failure": true } }, + { "rename":{ "field": "SN", "target_field": "certificate_surname", "ignore_failure": true } }, + { "rename":{ "field": "L", "target_field": "certificate_locality", "ignore_failure": true } }, + { "rename":{ "field": "GN", "target_field": "certificate_given_name", "ignore_failure": true } }, + { "rename":{ "field": "pseudonym", "target_field": "certificate_pseudonym", "ignore_failure": true } }, + { "rename":{ "field": "serialNumber", "target_field": "certificate_serial_number", "ignore_failure": true } }, + { "rename":{ "field": "title", "target_field": "certificate_title", "ignore_failure": true } }, + { "rename":{ "field": "initials", "target_field": "certificate_initials", "ignore_failure": true } }, + { "script":{ "lang": "painless", "source": "ctx.certificate_common_name_length = ctx.certificate_common_name.length()", "ignore_failure": true } }, + { "script":{ "lang": "painless", "source": "ctx.issuer_common_name_length = ctx.issuer_common_name.length()", "ignore_failure": true } }, + { "script":{ "lang": "painless", "source": "ctx.server_name_length = ctx.server_name.length()", "ignore_failure": true } }, + { + "pipeline": { + "name": "bro_common" + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_conn b/salt/elasticsearch/files/ingest/bro_conn new file mode 100644 index 000000000..b12be156e --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_conn @@ -0,0 +1,48 @@ +{ + "description" : "bro_conn", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.service", "target_field": "service", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_bytes", "target_field": "original_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_bytes", "target_field": "respond_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.conn_state", "target_field": "connection_state", "ignore_missing": true } }, + { "rename": { "field": "message2.local_orig", "target_field": "local_orig", "ignore_missing": true } }, + { "rename": { "field": "message2.local_resp", "target_field": "local_respond", "ignore_missing": true } }, + { "rename": { "field": "message2.missed_bytes", "target_field": "missed_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.history", "target_field": "history", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_pkts", "target_field": "original_packets", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_ip_bytes", "target_field": "original_ip_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_pkts", "target_field": "respond_packets", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_ip_bytes", "target_field": "respond_ip_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.tunnel_parents", "target_field": "tunnel_parents", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_cc", "target_field": "original_country_code","ignore_missing": true } }, + { "rename": { "field": "message2.resp_cc", "target_field": "respond_country_code", "ignore_missing": true } }, + { "rename": { "field": "message2.sensorname", "target_field": "sensor_name", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "ctx.total_bytes = (ctx.original_bytes + ctx.respond_bytes)", "ignore_failure": true } }, + { "set": { "if": "ctx.connection_state == 'S0'", "field": "connection_state_description", "value": "Connection attempt seen, no reply" } }, + { "set": { "if": "ctx.connection_state == 'S1'", "field": "connection_state_description", "value": "Connection established, not terminated" } }, + { "set": { "if": "ctx.connection_state == 'S2'", "field": "connection_state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } }, + { "set": { "if": "ctx.connection_state == 'S3'", "field": "connection_state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } }, + { "set": { "if": "ctx.connection_state == 'SF'", "field": "connection_state_description", "value": "Normal SYN/FIN completion" } }, + { "set": { "if": "ctx.connection_state == 'REJ'", "field": "connection_state_description", "value": "Connection attempt rejected" } }, + { "set": { "if": "ctx.connection_state == 'RSTO'", "field": "connection_state_description", "value": "Connection established, originator aborted (sent a RST)" } }, + { "set": { "if": "ctx.connection_state == 'RSTR'", "field": "connection_state_description", "value": "Established, responder aborted" } }, + { "set": { "if": "ctx.connection_state == 'RSTOS0'","field": "connection_state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } }, + { "set": { "if": "ctx.connection_state == 'RSTRH'", "field": "connection_state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } }, + { "set": { "if": "ctx.connection_state == 'SH'", "field": "connection_state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } }, + { "set": { "if": "ctx.connection_state == 'SHR'", "field": "connection_state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } }, + { "set": { "if": "ctx.connection_state == 'OTH'", "field": "connection_state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dce_rpc b/salt/elasticsearch/files/ingest/bro_dce_rpc new file mode 100644 index 000000000..105905245 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dce_rpc @@ -0,0 +1,20 @@ +{ + "description" : "bro_dce_rpc", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.rtt", "target_field": "rtt", "ignore_missing": true } }, + { "rename": { "field": "message2.named_pipe", "target_field": "named_pipe", "ignore_missing": true } }, + { "rename": { "field": "message2.endpoint", "target_field": "endpoint", "ignore_missing": true } }, + { "rename": { "field": "message2.operation", "target_field": "operation", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dhcp b/salt/elasticsearch/files/ingest/bro_dhcp new file mode 100644 index 000000000..010d0f85b --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dhcp @@ -0,0 +1,20 @@ +{ + "description" : "bro_dhcp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uids", "target_field": "uid", "ignore_missing": true } }, + { "rename": { "field": "message2.mac", "target_field": "mac", "ignore_missing": true } }, + { "rename": { "field": "message2.assigned_ip", "target_field": "assigned_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.lease_time", "target_field": "lease_time", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_id", "target_field": "transaction_id", "ignore_missing": true } }, + { "rename": { "field": "message2.assigned_addr", "target_field": "assigned_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.client_addr", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.server_addr", "target_field": "destination_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.requested_addr", "target_field": "requested_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.domain", "target_field": "domain_name", "ignore_missing": true } }, + { "rename": { "field": "message2.host_name", "target_field": "hostname", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.msg_types", "target_field": "message_types", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dnp3 b/salt/elasticsearch/files/ingest/bro_dnp3 new file mode 100644 index 000000000..bebb85ecb --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dnp3 @@ -0,0 +1,19 @@ +{ + "description" : "bro_dnp3", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fc_request", "target_field": "fc_request", "ignore_missing": true } }, + { "rename": { "field": "message2.fc_reply", "target_field": "fc_reply", "ignore_missing": true } }, + { "rename": { "field": "message2.iin", "target_field": "iin", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dns b/salt/elasticsearch/files/ingest/bro_dns new file mode 100644 index 000000000..be8d59294 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dns @@ -0,0 +1,35 @@ +{ + "description" : "bro_dns", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_id", "target_field": "transaction_id", "ignore_missing": true } }, + { "rename": { "field": "message2.rtt", "target_field": "rtt", "ignore_missing": true } }, + { "rename": { "field": "message2.query", "target_field": "query", "ignore_missing": true } }, + { "rename": { "field": "message2.qclass", "target_field": "query_class", "ignore_missing": true } }, + { "rename": { "field": "message2.qclass_name", "target_field": "query_class_name", "ignore_missing": true } }, + { "rename": { "field": "message2.qtype", "target_field": "query_type", "ignore_missing": true } }, + { "rename": { "field": "message2.qtype_name", "target_field": "query_type_name", "ignore_missing": true } }, + { "rename": { "field": "message2.rcode", "target_field": "rcode", "ignore_missing": true } }, + { "rename": { "field": "message2.rcode_name", "target_field": "rcode_name", "ignore_missing": true } }, + { "rename": { "field": "message2.AA", "target_field": "aa", "ignore_missing": true } }, + { "rename": { "field": "message2.TC", "target_field": "tc", "ignore_missing": true } }, + { "rename": { "field": "message2.RD", "target_field": "rd", "ignore_missing": true } }, + { "rename": { "field": "message2.RA", "target_field": "ra", "ignore_missing": true } }, + { "rename": { "field": "message2.Z", "target_field": "z", "ignore_missing": true } }, + { "rename": { "field": "message2.answers", "target_field": "answers", "ignore_missing": true } }, + { "rename": { "field": "message2.TTLs", "target_field": "ttls", "ignore_missing": true } }, + { "rename": { "field": "message2.rejected", "target_field": "rejected", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "ctx.query_length = ctx.query.length()", "ignore_failure": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dpd b/salt/elasticsearch/files/ingest/bro_dpd new file mode 100644 index 000000000..caf66d39e --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dpd @@ -0,0 +1,19 @@ +{ + "description" : "bro_dpd", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.analyzer", "target_field": "analyzer", "ignore_missing": true } }, + { "rename": { "field": "message2.failure_reason", "target_field": "failure_reason", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_files b/salt/elasticsearch/files/ingest/bro_files new file mode 100644 index 000000000..4337b75f2 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_files @@ -0,0 +1,32 @@ +{ + "description" : "bro_files", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.tx_hosts", "target_field": "file_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.rx_hosts.0", "target_field": "destination_ip", "ignore_missing": true } }, + { "remove": { "field": "message2.rx_hosts", "ignore_missing": true } }, + { "rename": { "field": "message2.conn_uids", "target_field": "uid", "ignore_missing": true } }, + { "remove": { "field": "source", "ignore_missing": true } }, + { "rename": { "field": "message2.source", "target_field": "source", "ignore_missing": true } }, + { "rename": { "field": "message2.depth", "target_field": "depth", "ignore_missing": true } }, + { "rename": { "field": "message2.analyzers", "target_field": "analyzer", "ignore_missing": true } }, + { "rename": { "field": "message2.mime_type", "target_field": "mimetype", "ignore_missing": true } }, + { "rename": { "field": "message2.filename", "target_field": "file_name", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.local_orig", "target_field": "local_orig", "ignore_missing": true } }, + { "rename": { "field": "message2.is_orig", "target_field": "is_orig", "ignore_missing": true } }, + { "rename": { "field": "message2.seen_bytes", "target_field": "seen_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.total_bytes", "target_field": "total_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.missing_bytes", "target_field": "missing_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.overflow_bytes", "target_field": "overflow_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.timedout", "target_field": "timed_out", "ignore_missing": true } }, + { "rename": { "field": "message2.parent_fuid", "target_field": "parent_fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.md5", "target_field": "md5", "ignore_missing": true } }, + { "rename": { "field": "message2.sha1", "target_field": "sha1", "ignore_missing": true } }, + { "rename": { "field": "message2.extracted", "target_field": "extracted", "ignore_missing": true } }, + { "rename": { "field": "message2.extracted_cutoff", "target_field": "extracted_cutoff", "ignore_missing": true } }, + { "rename": { "field": "message2.extracted_size", "target_field": "extracted_size", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ftp b/salt/elasticsearch/files/ingest/bro_ftp new file mode 100644 index 000000000..34775072d --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ftp @@ -0,0 +1,33 @@ +{ + "description" : "bro_http", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.user", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "message2.password", "target_field": "password", "ignore_missing": true } }, + { "rename": { "field": "message2.command", "target_field": "ftp_command", "ignore_missing": true } }, + { "rename": { "field": "message2.arg", "target_field": "ftp_argument", "ignore_missing": true } }, + { "rename": { "field": "message2.mime_type", "target_field": "mimetype", "ignore_missing": true } }, + { "rename": { "field": "message2.file_size", "target_field": "file_size", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_code", "target_field": "reply_code", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_msg", "target_field": "reply_message", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.passive", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.passive","target_field": "data_channel_passive", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.orig_h","target_field": "data_channel_source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.resp_h","target_field": "data_channel_destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.resp_p","target_field": "data_channel_destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_http b/salt/elasticsearch/files/ingest/bro_http new file mode 100644 index 000000000..842a12bc9 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_http @@ -0,0 +1,42 @@ +{ + "description" : "bro_http", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_depth", "target_field": "trans_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.method", "target_field": "method", "ignore_missing": true } }, + { "rename": { "field": "message2.host", "target_field": "virtual_host", "ignore_missing": true } }, + { "rename": { "field": "message2.uri", "target_field": "uri", "ignore_missing": true } }, + { "rename": { "field": "message2.referrer", "target_field": "referrer", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.user_agent", "target_field": "useragent", "ignore_missing": true } }, + { "rename": { "field": "message2.request_body_len", "target_field": "request_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.response_body_len","target_field": "response_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.status_code", "target_field": "status_code", "ignore_missing": true } }, + { "rename": { "field": "message2.status_msg", "target_field": "status_message", "ignore_missing": true } }, + { "rename": { "field": "message2.info_code", "target_field": "info_code", "ignore_missing": true } }, + { "rename": { "field": "message2.info_msg", "target_field": "info_message", "ignore_missing": true } }, + { "remove": { "field": "message2.tags", "ignore_failure": true } }, + { "rename": { "field": "message2.username", "target_field": "user", "ignore_missing": true } }, + { "rename": { "field": "message2.password", "target_field": "password", "ignore_missing": true } }, + { "rename": { "field": "message2.proxied", "target_field": "proxied", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_fuids", "target_field": "orig_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_filenames", "target_field": "orig_filenames", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_mime_types", "target_field": "orig_mime_types", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_fuids", "target_field": "resp_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_filenames", "target_field": "resp_filenames", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_mime_types", "target_field": "resp_mime_types", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", "ignore_failure": true } }, + { "script": { "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()", "ignore_failure": true } }, + { "script": { "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", "ignore_failure": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_intel b/salt/elasticsearch/files/ingest/bro_intel new file mode 100644 index 000000000..20bf90c5a --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_intel @@ -0,0 +1,29 @@ +{ + "description" : "bro_intel", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.indicator", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.indicator", "target_field": "indicator", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.indicator_type", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.indicator_type", "target_field": "indicator_type", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.where", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.where", "target_field": "seen_where", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.node", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.node", "target_field": "seen_node", "ignore_missing": true } }, + { "rename": { "field": "message2.matched", "target_field": "matched", "ignore_missing": true } }, + { "rename": { "field": "message2.sources", "target_field": "sources", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.file_mime_type", "target_field": "mimetype", "ignore_missing": true } }, + { "rename": { "field": "message2.file_desc", "target_field": "file_description", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_irc b/salt/elasticsearch/files/ingest/bro_irc new file mode 100644 index 000000000..c2a5ba22d --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_irc @@ -0,0 +1,25 @@ +{ + "description" : "bro_irc", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.nick", "target_field": "nick", "ignore_missing": true } }, + { "rename": { "field": "message2.user", "target_field": "irc_username", "ignore_missing": true } }, + { "rename": { "field": "message2.command", "target_field": "irc_command", "ignore_missing": true } }, + { "rename": { "field": "message2.value", "target_field": "value", "ignore_missing": true } }, + { "rename": { "field": "message2.addl", "target_field": "additional_info", "ignore_missing": true } }, + { "rename": { "field": "message2.dcc_file_name", "target_field": "dcc_file_name", "ignore_missing": true } }, + { "rename": { "field": "message2.dcc_file_size", "target_field": "dcc_file_size", "ignore_missing": true } }, + { "rename": { "field": "message2.dcc_mime_type", "target_field": "dcc_mime_type", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_kerberos b/salt/elasticsearch/files/ingest/bro_kerberos new file mode 100644 index 000000000..b338b5c96 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_kerberos @@ -0,0 +1,30 @@ +{ + "description" : "bro_kerberos", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.request_type", "target_field": "request_type", "ignore_missing": true } }, + { "rename": { "field": "message2.client", "target_field": "client", "ignore_missing": true } }, + { "rename": { "field": "message2.service", "target_field": "service", "ignore_missing": true } }, + { "rename": { "field": "message2.success", "target_field": "kerberos_success", "ignore_missing": true } }, + { "rename": { "field": "message2.error_msg", "target_field": "error_message", "ignore_missing": true } }, + { "rename": { "field": "message2.from", "target_field": "valid_from", "ignore_missing": true } }, + { "rename": { "field": "message2.till", "target_field": "valid_till", "ignore_missing": true } }, + { "rename": { "field": "message2.cipher", "target_field": "cipher", "ignore_missing": true } }, + { "rename": { "field": "message2.forwardable", "target_field": "forwardable", "ignore_missing": true } }, + { "rename": { "field": "message2.renewable", "target_field": "renewable", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_subject", "target_field": "client_certificate_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_fuid", "target_field": "client_certificate_fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.server_cert_subject", "target_field": "server_certificate_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.server_cert_fuid", "target_field": "server_certificate_fuid", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_modbus b/salt/elasticsearch/files/ingest/bro_modbus new file mode 100644 index 000000000..10e7c271a --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_modbus @@ -0,0 +1,18 @@ +{ + "description" : "bro_modbus", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.func", "target_field": "function", "ignore_missing": true } }, + { "rename": { "field": "message2.exception", "target_field": "exception", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_mysql b/salt/elasticsearch/files/ingest/bro_mysql new file mode 100644 index 000000000..a01d57da2 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_mysql @@ -0,0 +1,21 @@ +{ + "description" : "bro_mysql", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.cmd", "target_field": "mysql_command", "ignore_missing": true } }, + { "rename": { "field": "message2.arg", "target_field": "mysql_argument", "ignore_missing": true } }, + { "rename": { "field": "message2.success", "target_field": "mysql_success", "ignore_missing": true } }, + { "rename": { "field": "message2.rows", "target_field": "rows", "ignore_missing": true } }, + { "rename": { "field": "message2.response", "target_field": "response", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_notice b/salt/elasticsearch/files/ingest/bro_notice new file mode 100644 index 000000000..6e43448d5 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_notice @@ -0,0 +1,36 @@ +{ + "description" : "bro_notice", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "remove": { "field": "message2.dst", "ignore_failure": true } }, + { "remove": { "field": "message2.src", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.mime", "target_field": "file_mime_type", "ignore_missing": true } }, + { "rename": { "field": "message2.desc", "target_field": "file_description", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.note", "target_field": "note", "ignore_missing": true } }, + { "rename": { "field": "message2.msg", "target_field": "msg", "ignore_missing": true } }, + { "rename": { "field": "message2.sub", "target_field": "sub_msg", "ignore_missing": true } }, + { "rename": { "field": "message2.p", "target_field": "p", "ignore_missing": true } }, + { "rename": { "field": "message2.n", "target_field": "n", "ignore_missing": true } }, + { "rename": { "field": "message2.peer_descr", "target_field": "peer_description", "ignore_missing": true } }, + { "rename": { "field": "message2.actions", "target_field": "action", "ignore_missing": true } }, + { "rename": { "field": "message2.suppress_for", "target_field": "suppress_for", "ignore_missing": true } }, + { "rename": { "field": "message2.dropped", "target_field": "dropped", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_country_code", "target_field": "destination_country_code", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_region", "target_field": "destination_region", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_city", "target_field": "destination_city", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_latitude", "target_field": "destination_latitude", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_longitude", "target_field": "destination_longitude", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ntlm b/salt/elasticsearch/files/ingest/bro_ntlm new file mode 100644 index 000000000..a3d130343 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ntlm @@ -0,0 +1,24 @@ +{ + "description" : "bro_ntlm", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.hostname", "target_field": "hostname", "ignore_missing": true } }, + { "rename": { "field": "message2.domainname", "target_field": "domain_name", "ignore_missing": true } }, + { "rename": { "field": "message2.success", "target_field": "ntlm_success", "ignore_missing": true } }, + { "rename": { "field": "message2.status", "target_field": "status", "ignore_missing": true } }, + { "rename": { "field": "message2.username", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "message2.server_dns_computer_name", "target_field": "server_dns_computer_name", "ignore_missing": true } }, + { "rename": { "field": "message2.server_nb_computer_name", "target_field": "server_nb_computer_name", "ignore_missing": true } }, + { "rename": { "field": "message2.server_tree_name", "target_field": "server_tree_name", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_pe b/salt/elasticsearch/files/ingest/bro_pe new file mode 100644 index 000000000..2597d3e26 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_pe @@ -0,0 +1,23 @@ +{ + "description" : "bro_pe", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.machine", "target_field": "machine", "ignore_missing": true } }, + { "rename": { "field": "message2.compile_ts", "target_field": "compile_ts", "ignore_missing": true } }, + { "rename": { "field": "message2.os", "target_field": "os", "ignore_missing": true } }, + { "rename": { "field": "message2.subsystem", "target_field": "subsystem", "ignore_missing": true } }, + { "rename": { "field": "message2.is_exe", "target_field": "is_exe", "ignore_missing": true } }, + { "rename": { "field": "message2.is_64bit", "target_field": "is_64bit", "ignore_missing": true } }, + { "rename": { "field": "message2.uses_aslr", "target_field": "uses_aslr", "ignore_missing": true } }, + { "rename": { "field": "message2.uses_dep", "target_field": "uses_dep", "ignore_missing": true } }, + { "rename": { "field": "message2.uses_code_integrity","target_field": "uses_code_integrity","ignore_missing": true } }, + { "rename": { "field": "message2.uses_seh", "target_field": "uses_seh", "ignore_missing": true } }, + { "rename": { "field": "message2.has_import_table", "target_field": "has_import_table", "ignore_missing": true } }, + { "rename": { "field": "message2.has_export_table", "target_field": "has_export_table", "ignore_missing": true } }, + { "rename": { "field": "message2.has_cert_table", "target_field": "has_cert_table", "ignore_missing": true } }, + { "rename": { "field": "message2.has_debug_data", "target_field": "has_debug_data", "ignore_missing": true } }, + { "rename": { "field": "message2.section_names", "target_field": "section_names", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_radius b/salt/elasticsearch/files/ingest/bro_radius new file mode 100644 index 000000000..c333711d6 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_radius @@ -0,0 +1,25 @@ +{ + "description" : "bro_radius", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.username", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "message2.mac", "target_field": "mac", "ignore_missing": true } }, + { "rename": { "field": "message2.framed_addr", "target_field": "framed_addr", "ignore_missing": true } }, + { "rename": { "field": "message2.remote_ip", "target_field": "remote_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.connect_info", "target_field": "connect_info", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_msg", "target_field": "reply_message", "ignore_missing": true } }, + { "rename": { "field": "message2.result", "target_field": "result", "ignore_missing": true } }, + { "remove": { "field": "message2.ttl", "ignore_failure": true } }, + { "rename": { "field": "message2.logged", "target_field": "logged", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_rdp b/salt/elasticsearch/files/ingest/bro_rdp new file mode 100644 index 000000000..b3cf206a5 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_rdp @@ -0,0 +1,31 @@ +{ + "description" : "bro_rdp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.cookie", "target_field": "cookie", "ignore_missing": true } }, + { "rename": { "field": "message2.result", "target_field": "result", "ignore_missing": true } }, + { "rename": { "field": "message2.security_protocol","target_field": "security_protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.keyboard_layout", "target_field": "keyboard_layout", "ignore_missing": true } }, + { "rename": { "field": "message2.client_build", "target_field": "client_build", "ignore_missing": true } }, + { "rename": { "field": "message2.client_name", "target_field": "client_name", "ignore_missing": true } }, + { "rename": { "field": "message2.client_dig_product_id", "target_field": "client_digital_product_id", "ignore_missing": true } }, + { "rename": { "field": "message2.desktop_width", "target_field": "desktop_width", "ignore_missing": true } }, + { "rename": { "field": "message2.desktop_height", "target_field": "desktop_height", "ignore_missing": true } }, + { "rename": { "field": "message2.requested_color_depth", "target_field": "requested_color_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_type", "target_field": "certificate_type", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_count", "target_field": "certificate_count", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_permanent", "target_field": "certificate_permanent","ignore_missing": true } }, + { "rename": { "field": "message2.encryption_level", "target_field": "encryption_level", "ignore_missing": true } }, + { "rename": { "field": "message2.encryption_method","target_field": "encryption_method", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_rfb b/salt/elasticsearch/files/ingest/bro_rfb new file mode 100644 index 000000000..8f3cc86e7 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_rfb @@ -0,0 +1,26 @@ +{ + "description" : "bro_rfb", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.client_major_version", "target_field": "client_major_version", "ignore_missing": true } }, + { "rename": { "field": "message2.client_minor_version", "target_field": "client_minor_version", "ignore_missing": true } }, + { "rename": { "field": "message2.server_major_version", "target_field": "server_major_version", "ignore_missing": true } }, + { "rename": { "field": "message2.server_minor_version", "target_field": "server_minor_version", "ignore_missing": true } }, + { "rename": { "field": "message2.authentication_method", "target_field": "authentication_method","ignore_missing": true } }, + { "rename": { "field": "message2.auth", "target_field": "auth", "ignore_missing": true } }, + { "rename": { "field": "message2.share_flag", "target_field": "share_flag", "ignore_missing": true } }, + { "rename": { "field": "message2.desktop_name", "target_field": "desktop_name", "ignore_missing": true } }, + { "rename": { "field": "message2.width", "target_field": "width", "ignore_missing": true } }, + { "rename": { "field": "message2.height", "target_field": "height", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_signatures b/salt/elasticsearch/files/ingest/bro_signatures new file mode 100644 index 000000000..5dd3d9924 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_signatures @@ -0,0 +1,22 @@ +{ + "description" : "bro_signatures", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.note", "target_field": "note", "ignore_missing": true } }, + { "rename": { "field": "message2.sig_id", "target_field": "signature_id", "ignore_missing": true } }, + { "rename": { "field": "message2.event_msg", "target_field": "event_message", "ignore_missing": true } }, + { "rename": { "field": "message2.sub_msg", "target_field": "sub_message", "ignore_missing": true } }, + { "rename": { "field": "message2.sig_count", "target_field": "signature_count", "ignore_missing": true } }, + { "rename": { "field": "message2.host_count", "target_field": "host_count", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_sip b/salt/elasticsearch/files/ingest/bro_sip new file mode 100644 index 000000000..3a8b00d62 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_sip @@ -0,0 +1,37 @@ +{ + "description" : "bro_sip", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_depth", "target_field": "trans_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.method", "target_field": "method", "ignore_missing": true } }, + { "rename": { "field": "message2.uri", "target_field": "uri", "ignore_missing": true } }, + { "rename": { "field": "message2.date", "target_field": "date", "ignore_missing": true } }, + { "rename": { "field": "message2.request_from", "target_field": "request_from", "ignore_missing": true } }, + { "rename": { "field": "message2.request_to", "target_field": "request_to", "ignore_missing": true } }, + { "rename": { "field": "message2.response_from", "target_field": "response_from", "ignore_missing": true } }, + { "rename": { "field": "message2.response_to", "target_field": "response_to", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_to", "target_field": "reply_to", "ignore_missing": true } }, + { "rename": { "field": "message2.call_id", "target_field": "call_id", "ignore_missing": true } }, + { "rename": { "field": "message2.seq", "target_field": "seq", "ignore_missing": true } }, + { "rename": { "field": "message2.subject", "target_field": "subject", "ignore_missing": true } }, + { "rename": { "field": "message2.request_path", "target_field": "request_path", "ignore_missing": true } }, + { "rename": { "field": "message2.response_path", "target_field": "response_path", "ignore_missing": true } }, + { "rename": { "field": "message2.user_agent", "target_field": "user_agent", "ignore_missing": true } }, + { "rename": { "field": "message2.status_code", "target_field": "status_code", "ignore_missing": true } }, + { "rename": { "field": "message2.status_msg", "target_field": "status_msg", "ignore_missing": true } }, + { "rename": { "field": "message2.warning", "target_field": "warning", "ignore_missing": true } }, + { "rename": { "field": "message2.request_body_len", "target_field": "request_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.response_body_len","target_field": "response_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.content_type", "target_field": "content_type", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_smb_files b/salt/elasticsearch/files/ingest/bro_smb_files new file mode 100644 index 000000000..83ba8bd67 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_smb_files @@ -0,0 +1,31 @@ +{ + "description" : "bro_smb_files", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.action", "target_field": "action", "ignore_missing": true } }, + { "remove": { "field": "path", "ignore_failure": true } }, + { "rename": { "field": "message2.path", "target_field": "path", "ignore_missing": true } }, + { "rename": { "field": "message2.name", "target_field": "name", "ignore_missing": true } }, + { "rename": { "field": "message2.size", "target_field": "size", "ignore_missing": true } }, + { "rename": { "field": "message2.prev_name", "target_field": "prev_name", "ignore_missing": true } }, + { "dot_expander": { "field": "times.modified", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.modified", "target_field": "times_modified", "ignore_missing": true } }, + { "dot_expander": { "field": "times.accessed", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.accessed", "target_field": "times_accessed", "ignore_missing": true } }, + { "dot_expander": { "field": "times.created", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.created", "target_field": "times_created", "ignore_missing": true } }, + { "dot_expander": { "field": "times.changed", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.changed", "target_field": "times_changed", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_smb_mapping b/salt/elasticsearch/files/ingest/bro_smb_mapping new file mode 100644 index 000000000..e1b6b5dfb --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_smb_mapping @@ -0,0 +1,21 @@ +{ + "description" : "bro_smb_files", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "remove": { "field": "path", "ignore_failure": true } }, + { "rename": { "field": "message2.path", "target_field": "path", "ignore_missing": true } }, + { "rename": { "field": "message2.service", "target_field": "service", "ignore_missing": true } }, + { "rename": { "field": "message2.native_file_system", "target_field": "native_file_system", "ignore_missing": true } }, + { "rename": { "field": "message2.share_type", "target_field": "share_type", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_smtp b/salt/elasticsearch/files/ingest/bro_smtp new file mode 100644 index 000000000..4bd85a293 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_smtp @@ -0,0 +1,38 @@ +{ + "description" : "bro_smtp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "remove": { "field": "path", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_depth", "target_field": "trans_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.helo", "target_field": "helo", "ignore_missing": true } }, + { "rename": { "field": "message2.mailfrom", "target_field": "mail_from", "ignore_missing": true } }, + { "rename": { "field": "message2.rcptto", "target_field": "recipient_to", "ignore_missing": true } }, + { "rename": { "field": "message2.date", "target_field": "mail_date", "ignore_missing": true } }, + { "rename": { "field": "message2.from", "target_field": "from", "ignore_missing": true } }, + { "rename": { "field": "message2.to", "target_field": "to", "ignore_missing": true } }, + { "rename": { "field": "message2.cc", "target_field": "cc", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_to", "target_field": "reply_to", "ignore_missing": true } }, + { "rename": { "field": "message2.msg_id", "target_field": "message_id", "ignore_missing": true } }, + { "rename": { "field": "message2.in_reply_to", "target_field": "in_reply_to", "ignore_missing": true } }, + { "rename": { "field": "message2.subject", "target_field": "subject", "ignore_missing": true } }, + { "rename": { "field": "message2.x_originating_ip", "target_field": "x_originating_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.first_received", "target_field": "first_received", "ignore_missing": true } }, + { "rename": { "field": "message2.second_received", "target_field": "second_received", "ignore_missing": true } }, + { "rename": { "field": "message2.last_reply", "target_field": "last_reply", "ignore_missing": true } }, + { "rename": { "field": "message2.path", "target_field": "path", "ignore_missing": true } }, + { "rename": { "field": "message2.user_agent", "target_field": "useragent", "ignore_missing": true } }, + { "rename": { "field": "message2.tls", "target_field": "tls", "ignore_missing": true } }, + { "rename": { "field": "message2.fuids", "target_field": "fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.is_webmail", "target_field": "is_webmail", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_snmp b/salt/elasticsearch/files/ingest/bro_snmp new file mode 100644 index 000000000..bec88c1af --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_snmp @@ -0,0 +1,25 @@ +{ + "description" : "bro_snmp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.community", "target_field": "community", "ignore_missing": true } }, + { "rename": { "field": "message2.get_requests", "target_field": "get_requests", "ignore_missing": true } }, + { "rename": { "field": "message2.get_bulk_requests","target_field": "get_bulk_requests", "ignore_missing": true } }, + { "rename": { "field": "message2.get_responses", "target_field": "get_responses", "ignore_missing": true } }, + { "rename": { "field": "message2.set_requests", "target_field": "set_requests", "ignore_missing": true } }, + { "rename": { "field": "message2.display_string", "target_field": "display_string", "ignore_missing": true } }, + { "rename": { "field": "message2.up_since", "target_field": "up_since", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_socks b/salt/elasticsearch/files/ingest/bro_socks new file mode 100644 index 000000000..38c5dd528 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_socks @@ -0,0 +1,28 @@ +{ + "description" : "bro_socks", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.user", "target_field": "user", "ignore_missing": true } }, + { "rename": { "field": "message2.password", "target_field": "password", "ignore_missing": true } }, + { "rename": { "field": "message2.status", "target_field": "status", "ignore_missing": true } }, + { "rename": { "field": "message2.request_host", "target_field": "request_host", "ignore_missing": true } }, + { "dot_expander": { "field": "request.name", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.request.name", "target_field": "request_name", "ignore_missing": true } }, + { "rename": { "field": "message2.request_p", "target_field": "request_port", "ignore_missing": true } }, + { "dot_expander": { "field": "bound.host", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.bound.host", "target_field": "bound_host", "ignore_missing": true } }, + { "rename": { "field": "message2.bound_name", "target_field": "bound_name", "ignore_missing": true } }, + { "rename": { "field": "message2.bound_p", "target_field": "bound_port", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_software b/salt/elasticsearch/files/ingest/bro_software new file mode 100644 index 000000000..e742fda9e --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_software @@ -0,0 +1,23 @@ +{ + "description" : "bro_software", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "version.major", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.major", "target_field": "version_major", "ignore_missing": true } }, + { "dot_expander": { "field": "version.minor", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.minor", "target_field": "version_minor", "ignore_missing": true } }, + { "dot_expander": { "field": "version.minor2", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.minor2", "target_field": "version_minor2", "ignore_missing": true } }, + { "dot_expander": { "field": "version.minor3", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.minor3", "target_field": "version_minor3", "ignore_missing": true } }, + { "dot_expander": { "field": "version.addl", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.addl", "target_field": "version_additional_info", "ignore_missing": true } }, + { "rename": { "field": "message2.host", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.host_p", "target_field": "source_port", "ignore_missing": true } }, + { "rename": { "field": "message2.software_type", "target_field": "software_type", "ignore_missing": true } }, + { "rename": { "field": "message2.name", "target_field": "name", "ignore_missing": true } }, + { "rename": { "field": "message2.unparsed_version", "target_field": "unparsed_version", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ssh b/salt/elasticsearch/files/ingest/bro_ssh new file mode 100644 index 000000000..7df949503 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ssh @@ -0,0 +1,40 @@ +{ + "description" : "bro_conn", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "rename": { "field": "message2.hassh", "target_field": "hassh", "ignore_missing": true } }, + { "rename": { "field": "message2.auth_success", "target_field": "authentication_success", "ignore_missing": true } }, + { "rename": { "field": "message2.auth_attempts", "target_field": "authentication_attempts", "ignore_missing": true } }, + { "rename": { "field": "message2.direction", "target_field": "direction", "ignore_missing": true } }, + { "rename": { "field": "message2.client", "target_field": "client", "ignore_missing": true } }, + { "rename": { "field": "message2.server", "target_field": "server", "ignore_missing": true } }, + { "rename": { "field": "message2.cipher_alg", "target_field": "cipher_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.compression_alg", "target_field": "compression_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.cshka", "target_field": "client_host_key_algorithms", "ignore_missing": true } }, + { "rename": { "field": "message2.host_key_alg", "target_field": "host_key_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshAlgorithms", "target_field": "hassh_algorithms", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshServer", "target_field": "hassh_server", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshVersion", "target_field": "hassh_version", "ignore_missing": true } }, + { "rename": { "field": "message2.kex_alg", "target_field": "kex_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.mac_alg", "target_field": "mac_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.sshka", "target_field": "server_host_key_algorithms", "ignore_missing": true } }, + { "rename": { "field": "message2.host_key", "target_field": "host_key", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_region", "target_field": "destination_region", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_city", "target_field": "destination_city", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_latitude", "target_field": "destination_latitude", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_longitude", "target_field": "destination_longitude", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_country_code", "target_field": "destination_country_code", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshServerAlgorithms", "target_field": "hassh_server_algorithms", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ssl b/salt/elasticsearch/files/ingest/bro_ssl new file mode 100644 index 000000000..04d0fc8ec --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ssl @@ -0,0 +1,33 @@ +{ + "description" : "bro_ssl", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.cipher", "target_field": "cipher", "ignore_missing": true } }, + { "rename": { "field": "message2.curve", "target_field": "curve", "ignore_missing": true } }, + { "rename": { "field": "message2.server_name", "target_field": "server_name", "ignore_missing": true } }, + { "rename": { "field": "message2.resumed", "target_field": "resumed", "ignore_missing": true } }, + { "rename": { "field": "message2.last_alert", "target_field": "last_alert", "ignore_missing": true } }, + { "rename": { "field": "message2.next_protocol", "target_field": "next_protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.established", "target_field": "established", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_chain_fuids", "target_field": "certificate_chain_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_chain_fuids", "target_field": "client_certificate_chain_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.subject", "target_field": "certificate_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.issuer", "target_field": "certificate_issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.client_subject", "target_field": "client_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.client_issuer", "target_field": "client_issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.validation_status","target_field": "validation_status", "ignore_missing": true } }, + { "rename": { "field": "message2.ja3", "target_field": "ja3", "ignore_missing": true } }, + { "rename": { "field": "message2.ja3s", "target_field": "ja3s", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common_ssl" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_syslog b/salt/elasticsearch/files/ingest/bro_syslog new file mode 100644 index 000000000..9599b435c --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_syslog @@ -0,0 +1,21 @@ +{ + "description" : "bro_syslog", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.facility", "target_field": "facility", "ignore_missing": true } }, + { "rename": { "field": "message2.severity", "target_field": "severity", "ignore_missing": true } }, + { "remove": { "field": "message", "ignore_failure": true } }, + { "rename": { "field": "message2.message", "target_field": "message", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_tunnel b/salt/elasticsearch/files/ingest/bro_tunnel new file mode 100644 index 000000000..21fa06deb --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_tunnel @@ -0,0 +1,7 @@ +{ + "description" : "bro_tunnel", + "processors" : [ + { "set": { "field": "event_type", "value": "bro_tunnels" } }, + { "pipeline": { "name": "bro_tunnels" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_tunnels b/salt/elasticsearch/files/ingest/bro_tunnels new file mode 100644 index 000000000..50c12518f --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_tunnels @@ -0,0 +1,18 @@ +{ + "description" : "bro_tunnels", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.tunnel_type", "target_field": "tunnel_type", "ignore_missing": true } }, + { "rename": { "field": "message2.action", "target_field": "action", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_weird b/salt/elasticsearch/files/ingest/bro_weird new file mode 100644 index 000000000..b471f5e75 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_weird @@ -0,0 +1,20 @@ +{ + "description" : "bro_weird", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.name", "target_field": "name", "ignore_missing": true } }, + { "rename": { "field": "message2.addl", "target_field": "additional_info", "ignore_missing": true } }, + { "rename": { "field": "message2.notice", "target_field": "notice", "ignore_missing": true } }, + { "rename": { "field": "message2.peer", "target_field": "peer", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_x509 b/salt/elasticsearch/files/ingest/bro_x509 new file mode 100644 index 000000000..56e905347 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_x509 @@ -0,0 +1,44 @@ +{ + "description" : "bro_x509", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id", "target_field": "id", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.version", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.version", "target_field": "certificate_version", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.serial", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.serial", "target_field": "certificate_serial", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.subject", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.subject", "target_field": "certificate_subject", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.issuer", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.issuer", "target_field": "certificate_issuer", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.not_valid_before", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.not_valid_before", "target_field": "certificate_not_valid_before", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.not_valid_after", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.not_valid_after", "target_field": "certificate_not_valid_after", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_alg", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_alg", "target_field": "certificate_key_algorithm", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.sig_alg", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.sig_alg", "target_field": "certificate_signing_algorithm", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_type", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_type", "target_field": "certificate_key_type", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_length", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_length", "target_field": "certificate_key_length", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.exponent", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.exponent", "target_field": "certificate_exponent", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.curve", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.curve", "target_field": "certificate_curve", "ignore_missing": true } }, + { "dot_expander": { "field": "san.dns", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.dns", "target_field": "san_dns", "ignore_missing": true } }, + { "dot_expander": { "field": "san.uri", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.uri", "target_field": "san_uri", "ignore_missing": true } }, + { "dot_expander": { "field": "san.email", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.email", "target_field": "san_email", "ignore_missing": true } }, + { "dot_expander": { "field": "san.ip", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.ip", "target_field": "san_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "basic_constraints.ca", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.basic_constraints.ca", "target_field": "basic_constraints_ca", "ignore_missing": true } }, + { "dot_expander": { "field": "basic_constraints.path_length", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.basic_constraints.path_length", "target_field": "basic_constraints_path_length", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common_ssl" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common new file mode 100644 index 000000000..ed227258e --- /dev/null +++ b/salt/elasticsearch/files/ingest/common @@ -0,0 +1,52 @@ +{ + "description" : "common", + "processors" : [ + { + "rename": { + "field": "type", + "target_field": "event_type", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "destination_ip", + "target_field": "destination_geo", + "database_file": "GeoLite2-City.mmdb", + "ignore_missing": true, + "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] + } + }, + { + "geoip": { + "field": "source_ip", + "target_field": "source_geo", + "database_file": "GeoLite2-City.mmdb", + "ignore_missing": true, + "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] + } + }, + { + "split": { + "field": "_index", + "target_field": "index_name_prefix", + "separator": "-" + } + }, + { + "date_index_name": { + "field": "@timestamp", + "index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-", + "date_rounding": "d", + "ignore_failure": true, + "index_name_format": "yyyy.MM.dd" + } + }, + { + "remove": { + "field": "index_name_prefix", + "ignore_failure": true + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/common_nids b/salt/elasticsearch/files/ingest/common_nids new file mode 100644 index 000000000..4fffab7c1 --- /dev/null +++ b/salt/elasticsearch/files/ingest/common_nids @@ -0,0 +1,17 @@ +{ + "description" : "common_nids", + "processors" : [ + { "convert": { "field": "sid", "type": "integer" } }, + { "set": { "if": "ctx.sid < 1000000", "field": "signature_info", "value": "https://www.snort.org/search?query={{gid}}-{{sid}}" } }, + { "set": { "if": "ctx.sid > 1999999", "field": "signature_info", "value": "https://doc.emergingthreats.net/{{sid}}" } }, + { "remove": { "if": "ctx.sid > 2999999", "field": "signature_info" } }, + { "set": { "if": "ctx.priority == '1'", "field": "severity", "value": "High" } }, + { "set": { "if": "ctx.priority == '2'", "field": "severity", "value": "Medium" } }, + { "set": { "if": "ctx.priority == '3'", "field": "severity", "value": "Low" } }, + { "dissect": { "field": "alert", "pattern" : "%{rule_type} %{category} ", "ignore_failure": true } }, + { "set": { "if": "ctx.rule_type == 'GPL'", "field": "rule_type", "value": "Snort GPL" } }, + { "set": { "if": "ctx.rule_type == 'ET'", "field": "rule_type", "value": "Emerging Threats" } }, + { "lowercase": { "field": "category", "ignore_failure": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/ossec b/salt/elasticsearch/files/ingest/ossec new file mode 100644 index 000000000..2b6d19370 --- /dev/null +++ b/salt/elasticsearch/files/ingest/ossec @@ -0,0 +1,53 @@ +{ + "description" : "ossec", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.agent", "target_field": "agent", "ignore_missing": true } }, + { "rename": { "field": "message2.data", "target_field": "data", "ignore_missing": true } }, + { "rename": { "field": "message2.decoder", "target_field": "decoder", "ignore_missing": true } }, + { "rename": { "field": "message2.full_log", "target_field": "full_log", "ignore_missing": true } }, + { "rename": { "field": "message2.id", "target_field": "id", "ignore_missing": true } }, + { "rename": { "field": "message2.location", "target_field": "location", "ignore_missing": true } }, + { "rename": { "field": "message2.manager", "target_field": "manager", "ignore_missing": true } }, + { "rename": { "field": "message2.predecoder", "target_field": "predecoder", "ignore_missing": true } }, + { "rename": { "field": "message2.timestamp", "target_field": "timestamp", "ignore_missing": true } }, + { "rename": { "field": "message2.rule", "target_field": "wazuh-rule", "ignore_missing": true } }, + { "rename": { "field": "data.command", "target_field": "command", "ignore_missing": true } }, + { "rename": { "field": "data.dstip", "target_field": "destination_ip", "ignore_missing": true } }, + { "rename": { "field": "data.dstport", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "data.dstuser", "target_field": "escalated_user", "ignore_missing": true } }, + { "rename": { "field": "data.srcip", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "data.srcuser", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationHostname", "target_field": "destination_hostname", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationIp", "target_field": "destination_ip", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationPort", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.image", "target_field": "image_path", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.parentImage", "target_field": "parent_image_path", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourceHostname", "target_field": "source_hostname", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourceIp", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourcePort", "target_field": "source_port", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.targetFilename", "target_field": "target_filename", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.user", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "data.win.system.eventID", "target_field": "event_id", "ignore_missing": true } }, + { "rename": { "field": "predecoder.program_name", "target_field": "process", "ignore_missing": true } }, + { "rename": { "field": "wazuh-rule.level", "target_field": "alert_level", "ignore_missing": true } }, + { "rename": { "field": "wazuh-rule.description", "target_field": "description", "ignore_missing": true } }, + { "set": { "if": "ctx.alert_level == 1", "field": "classification", "value": "None" } }, + { "set": { "if": "ctx.alert_level == 2", "field": "classification", "value": "System low priority notification" } }, + { "set": { "if": "ctx.alert_level == 3", "field": "classification", "value": "Successful/authorized event" } }, + { "set": { "if": "ctx.alert_level == 4", "field": "classification", "value": "System low priority error" } }, + { "set": { "if": "ctx.alert_level == 5", "field": "classification", "value": "User generated error" } }, + { "set": { "if": "ctx.alert_level == 6", "field": "classification", "value": "Low relevance attack" } }, + { "set": { "if": "ctx.alert_level == 7", "field": "classification", "value": "\"Bad word\" matching" } }, + { "set": { "if": "ctx.alert_level == 8", "field": "classification", "value": "First time seen" } }, + { "set": { "if": "ctx.alert_level == 9", "field": "classification", "value": "Error from invalid source" } }, + { "set": { "if": "ctx.alert_level == 10", "field": "classification", "value": "Multiple user generated errors" } }, + { "set": { "if": "ctx.alert_level == 11", "field": "classification", "value": "Integrity checking warning" } }, + { "set": { "if": "ctx.alert_level == 12", "field": "classification", "value": "High importance event" } }, + { "set": { "if": "ctx.alert_level == 13", "field": "classification", "value": "Unusal error (high importance)" } }, + { "set": { "if": "ctx.alert_level == 14", "field": "classification", "value": "High importance security event" } }, + { "set": { "if": "ctx.alert_level == 15", "field": "classification", "value": "Severe attack" } }, + { "append": { "if": "ctx.alert_level != null", "field": "tags", "value": ["alert"] } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/sguild_nids b/salt/elasticsearch/files/ingest/sguild_nids new file mode 100644 index 000000000..c7bcdc418 --- /dev/null +++ b/salt/elasticsearch/files/ingest/sguild_nids @@ -0,0 +1,25 @@ +{ + "description" : "sguild_nids", + "processors" : [ + { + "dissect": { + "field": "message", + "pattern" : "%{} %{} %{} Alert Received: %{} %{priority} %{classification} %{interface} {%{alerttime}} %{} %{} {%{alert}} %{source_ip} %{destination_ip} %{protocol} %{source_port} %{destination_port} %{gid} %{sid} %{rev} ", + "on_failure": [ { "drop" : { } } ] + } + }, + { "set": { "if": "ctx.protocol == '1'", "field": "protocol", "value": "ICMP" } }, + { "set": { "if": "ctx.protocol == '6'", "field": "protocol", "value": "TCP" } }, + { "set": { "if": "ctx.protocol == '17'", "field": "protocol", "value": "UDP" } }, + { "remove": { "if": "ctx.source_ip == '{}'", "field": "source_ip" } }, + { "remove": { "if": "ctx.destination_ip == '{}'", "field": "destination_ip" } }, + { "remove": { "if": "ctx.protocol == '{}'", "field": "protocol" } }, + { "remove": { "if": "ctx.source_port == '{}'", "field": "source_port" } }, + { "remove": { "if": "ctx.destination_port == '{}'", "field": "destination_port" } }, + { "set": { "field": "type", "value": "snort" } }, + { "rename": { "field": "@timestamp", "target_field": "timestamp", "ignore_missing": true } }, + { "date": { "field": "alerttime", "target_field": "@timestamp", "formats": ["yyyy-MM-dd HH:mm:ss"], "ignore_failure": true } }, + { "remove": { "field": "alerttime", "ignore_missing": true } }, + { "pipeline": { "name": "common_nids" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/snort b/salt/elasticsearch/files/ingest/snort new file mode 100644 index 000000000..b841ca917 --- /dev/null +++ b/salt/elasticsearch/files/ingest/snort @@ -0,0 +1,21 @@ +{ + "description" : "snort", + "processors" : [ + { + "dissect": { + "field": "message", + "pattern" : "[%{gid}:%{sid}:%{rev}] %{alert} [Classification: %{classification}] [Priority: %{priority}]: <%{interface}> {%{protocol}} %{source_ip_port} -> %{destination_ip_port}", + "on_failure": [ { "drop" : { } } ] + } + }, + { "split": { "field": "source_ip_port", "separator": ":", "ignore_failure": true } }, + { "split": { "field": "destination_ip_port", "separator": ":", "ignore_failure": true } }, + { "rename":{ "field": "source_ip_port.1", "target_field": "source_port", "ignore_failure": true } }, + { "rename":{ "field": "destination_ip_port.1", "target_field": "destination_port", "ignore_failure": true } }, + { "rename":{ "field": "source_ip_port.0", "target_field": "source_ip", "ignore_failure": true } }, + { "rename":{ "field": "destination_ip_port.0", "target_field": "destination_ip", "ignore_failure": true } }, + { "remove":{ "field": "source_ip_port", "ignore_failure": true } }, + { "remove":{ "field": "destination_ip_port", "ignore_failure": true } }, + { "pipeline": { "name": "common_nids" } } + ] +} diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines new file mode 100755 index 000000000..c0dd44aa9 --- /dev/null +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +ELASTICSEARCH_HOST=$1 +ELASTICSEARCH_PORT=9200 + +# Define a default directory to load pipelines from +ELASTICSEARCH_INGEST_PIPELINES="/opt/so/saltstack/salt/elasticsearch/files/ingest/" + +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo +fi + +cd ${ELASTICSEARCH_INGEST_PIPELINES} + +echo "Loading pipelines..." +for i in *; do echo $i; curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +echo + +cd - >/dev/null diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index c05cb83e4..1eb3a19e4 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -121,6 +121,10 @@ so-elasticsearch: - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw +so-elasticsearch-pipelines: + cmd.run: + - name: /opt/so/saltstack/salt/elasticsearch/files/so-elasticsearch-pipelines {{ esclustername }} + # Tell the main cluster I am here #curl -XPUT http://\$ELASTICSEARCH_HOST:\$ELASTICSEARCH_PORT/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"$HOSTNAME": {"skip_unavailable": "true", "seeds": ["$DOCKER_INTERFACE:$REVERSE_PORT"]}}}}}' diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 67fd596c5..f0d3a8587 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -6,6 +6,59 @@ name: {{ HOSTNAME }} +#================================ Logging ====================================== +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +logging.level: debug + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, filebeat periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + path: /usr/share/filebeat/logs + + # The name of the files where the logs are written to. + name: filebeat.log + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + +# Set to true to log messages in json format. +#logging.json: false + #========================== Modules configuration ============================ filebeat.modules: #=========================== Filebeat prospectors ============================= @@ -181,58 +234,6 @@ output.logstash: -#================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -logging.level: debug - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, filebeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/filebeat - - # The name of the files where the logs are written to. - name: filebeat.log - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 7 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false #============================== Xpack Monitoring ===================================== diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index f14e71772..b92899ef0 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -23,6 +23,13 @@ filebeatetcdir: - group: 939 - makedirs: True +filebeatlogdir: + file.directory: + - name: /opt/so/log/filebeat + - user: 939 + - group: 939 + - makedirs: True + filebeatpkidir: file.directory: - name: /opt/so/conf/filebeat/etc/pki @@ -51,18 +58,18 @@ filebeatconfsync: so-filebeatimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-filebeat:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-filebeat:HH1.1.1 so-filebeat: docker_container.running: - require: - so-filebeatimage - - image: soshybridhunter/so-filebeat:HH1.1.0 + - image: soshybridhunter/so-filebeat:HH1.1.1 - hostname: so-filebeat - user: root - extra_hosts: {{ MASTER }}:{{ MASTERIP }} - binds: - - /opt/so/log/filebeat:/var/log/filebeat:rw + - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - /nsm/bro:/nsm/bro:ro - /opt/so/log/suricata:/suricata:ro diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 7044699f0..2489b1f47 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -228,6 +228,28 @@ enable_master_osquery_8080_{{ip}}: - position: 1 - save: True +enable_master_playbook_3200_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 3200 + - position: 1 + - save: True + +enable_master_navigator_4200_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 4200 + - position: 1 + - save: True + {% endfor %} # Make it so all the minions can talk to salt and update etc. diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 3fc364e4e..5897f6a93 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -33,13 +33,13 @@ hiveesdata: so-thehive-esimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive-es:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive-es:HH1.1.1 so-thehive-es: docker_container.running: - require: - so-thehive-esimage - - image: soshybridhunter/so-thehive-es:HH1.1.0 + - image: soshybridhunter/so-thehive-es:HH1.1.1 - hostname: so-thehive-es - name: so-thehive-es - user: 939 @@ -56,7 +56,6 @@ so-thehive-es: - transport.tcp.port=9500 - transport.host=0.0.0.0 - cluster.name=hive - - script.inline=true - thread_pool.index.queue_size=100000 - thread_pool.search.queue_size=100000 - thread_pool.bulk.queue_size=100000 @@ -81,13 +80,13 @@ so-thehive-es: so-thehiveimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive:HH1.1.1 so-thehive: docker_container.running: - require: - so-thehiveimage - - image: soshybridhunter/so-thehive:HH1.1.0 + - image: soshybridhunter/so-thehive:HH1.1.1 - environment: - ELASTICSEARCH_HOST={{ MASTERIP }} - hostname: so-thehive @@ -97,8 +96,9 @@ so-thehive: - /opt/so/conf/hive/etc/application.conf:/opt/thehive/conf/application.conf:ro - port_bindings: - 0.0.0.0:9000:9000 - + hivescript: cmd.script: - source: salt://hive/thehive/scripts/hive_init.sh + - cwd: /opt/so - template: jinja diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf index 1fd4b4816..e4dd1e2b2 100644 --- a/salt/hive/thehive/etc/application.conf +++ b/salt/hive/thehive/etc/application.conf @@ -5,59 +5,57 @@ # WARNING: If you deploy your application on several servers, make sure to use the same key. play.http.secret.key="letsdewdis" play.http.context=/thehive/ - +search.uri = "http://{{ MASTERIP }}:9400" # Elasticsearch search { - ## Basic configuration - # Index name. - index = the_hive - # ElasticSearch cluster name. - cluster = hive - # ElasticSearch instance address. - host = ["{{ MASTERIP }}:9500"] + # Name of the index + index = the_hive + # Name of the Elasticsearch cluster + cluster = hive + # Address of the Elasticsearch instance + host = ["{{ MASTERIP }}:9500"] + #search.uri = "http://{{ MASTERIP }}:9500" + # Scroll keepalive + keepalive = 1m + # Size of the page for scroll + pagesize = 50 + # Number of shards + nbshards = 5 + # Number of replicas + nbreplicas = 1 + # Arbitrary settings + settings { + # Maximum number of nested fields + mapping.nested_fields.limit = 100 + } - ## Advanced configuration - # Scroll keepalive. - #keepalive = 1m - # Scroll page size. - #pagesize = 50 - # Number of shards - #nbshards = 5 - # Number of replicas - #nbreplicas = 1 - # Arbitrary settings - #settings { - # # Maximum number of nested fields - # mapping.nested_fields.limit = 100 - #} + ### XPack SSL configuration + # Username for XPack authentication + #username + # Password for XPack authentication + #password + # Enable SSL to connect to ElasticSearch + ssl.enabled = false + # Path to certificate authority file + #ssl.ca + # Path to certificate file + #ssl.certificate + # Path to key file + #ssl.key - ### XPack SSL configuration - # Username for XPack authentication - #search.username = "" - # Password for XPack authentication - #search.password = "" - # Enable SSL to connect to ElasticSearch - search.ssl.enabled = false - # Path to certificate authority file - #search.ssl.ca = "" - # Path to certificate file - #search.ssl.certificate = "" - # Path to key file - #search.ssl.key = "" - - ### SearchGuard configuration - # Path to JKS file containing client certificate - #search.guard.keyStore.path = "" - # Password of the keystore - #search.guard.keyStore.password = "" - # Path to JKS file containing certificate authorities - #search.guard.trustStore.path = "" - ## Password of the truststore - #search.guard.trustStore.password = "" - # Enforce hostname verification - #search.guard.hostVerification = false - # If hostname verification is enabled specify if hostname should be resolved - #search.guard.hostVerificationResolveHostname = false + ### SearchGuard configuration + # Path to JKS file containing client certificate + #guard.keyStore.path + # Password of the keystore + #guard.keyStore.password + # Path to JKS file containing certificate authorities + #guard.trustStore.path + ## Password of the truststore + #guard.trustStore.password + # Enforce hostname verification + #guard.hostVerification + # If hostname verification is enabled specify if hostname should be resolved + #guard.hostVerificationResolveHostname } # Authentication diff --git a/salt/hive/thehive/etc/es/elasticsearch.yml b/salt/hive/thehive/etc/es/elasticsearch.yml index d600830b6..d00c01d5d 100644 --- a/salt/hive/thehive/etc/es/elasticsearch.yml +++ b/salt/hive/thehive/etc/es/elasticsearch.yml @@ -11,7 +11,6 @@ http.host: 0.0.0.0 http.port: 9400 transport.tcp.port: 9500 transport.host: 0.0.0.0 -script.inline: true thread_pool.index.queue_size: 100000 thread_pool.search.queue_size: 100000 thread_pool.bulk.queue_size: 100000 diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 7a403fd11..26910b5b0 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -56,14 +56,14 @@ synckibanacustom: so-kibanaimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-kibana:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-kibana:HH1.1.1 # Start the kibana docker so-kibana: docker_container.running: - require: - so-kibanaimage - - image: soshybridhunter/so-kibana:HH1.1.0 + - image: soshybridhunter/so-kibana:HH1.1.1 - hostname: kibana - user: kibana - environment: diff --git a/salt/logstash/conf/conf.enabled.txt.parser b/salt/logstash/conf/conf.enabled.txt.parser index a34b39c5f..6fbf3ba45 100644 --- a/salt/logstash/conf/conf.enabled.txt.parser +++ b/salt/logstash/conf/conf.enabled.txt.parser @@ -7,11 +7,12 @@ # /usr/share/logstash/pipeline.custom/1234_input_custom.conf ## # All of the defaults are loaded. +# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest. /usr/share/logstash/pipeline.dynamic/0900_input_redis.conf /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf /usr/share/logstash/pipeline.so/1002_preprocess_json.conf -/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf /usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf /usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf @@ -20,44 +21,44 @@ /usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf /usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf -/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf -/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf -/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf -/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf -/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf -/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf -/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf -/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf -/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf -/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf -/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf -/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf -/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf -/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf -/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf -/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf -/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf -/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf -/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf -/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf -/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf -/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf -/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf -/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf -/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf -/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf -/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf -/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf -/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf -/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf -/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf -/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf -/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf -/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf /usr/share/logstash/pipeline.so/1998_test_data.conf /usr/share/logstash/pipeline.so/2000_network_flow.conf -/usr/share/logstash/pipeline.so/6000_bro.conf -/usr/share/logstash/pipeline.so/6001_bro_import.conf +#/usr/share/logstash/pipeline.so/6000_bro.conf +#/usr/share/logstash/pipeline.so/6001_bro_import.conf /usr/share/logstash/pipeline.so/6002_syslog.conf /usr/share/logstash/pipeline.so/6101_switch_brocade.conf /usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf @@ -68,17 +69,17 @@ /usr/share/logstash/pipeline.so/6500_ossec.conf /usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf -/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf /usr/share/logstash/pipeline.so/8006_postprocess_dns.conf /usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf /usr/share/logstash/pipeline.so/8007_postprocess_http.conf /usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf /usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf -/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf -/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf -/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf -/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf +#/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf +#/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf +#/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf +#/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf /usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf /usr/share/logstash/pipeline.dynamic/9999_output_redis.conf diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index e7680b3be..d125fc829 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -7,6 +7,7 @@ # /usr/share/logstash/pipeline.custom/1234_input_custom.conf ## # All of the defaults are loaded. +# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest. #/usr/share/logstash/pipeline.so/0000_input_syslogng.conf #/usr/share/logstash/pipeline.so/0001_input_json.conf #/usr/share/logstash/pipeline.so/0002_input_windows_json.conf @@ -18,7 +19,7 @@ #/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf #/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf #/usr/share/logstash/pipeline.so/1002_preprocess_json.conf -/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf #/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf #/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf @@ -27,44 +28,44 @@ #/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf #/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf -/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf -/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf -/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf -/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf -/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf -/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf -/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf -/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf -/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf -/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf -/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf -/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf -/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf -/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf -/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf -/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf -/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf -/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf -/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf -/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf -/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf -/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf -/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf -/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf -/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf -/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf -/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf -/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf -/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf -/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf -/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf -/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf -/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf -/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf #/usr/share/logstash/pipeline.so/1998_test_data.conf #/usr/share/logstash/pipeline.so/2000_network_flow.conf -/usr/share/logstash/pipeline.so/6000_bro.conf -/usr/share/logstash/pipeline.so/6001_bro_import.conf +#/usr/share/logstash/pipeline.so/6000_bro.conf +#/usr/share/logstash/pipeline.so/6001_bro_import.conf #/usr/share/logstash/pipeline.so/6002_syslog.conf #/usr/share/logstash/pipeline.so/6101_switch_brocade.conf #/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf @@ -77,7 +78,7 @@ /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf /usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf /usr/share/logstash/pipeline.so/6700_winlogbeat.conf -/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf #/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf #/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf diff --git a/salt/logstash/conf/conf.enabled.txt.storage b/salt/logstash/conf/conf.enabled.txt.storage index 6d2d581db..470f19c55 100644 --- a/salt/logstash/conf/conf.enabled.txt.storage +++ b/salt/logstash/conf/conf.enabled.txt.storage @@ -7,11 +7,12 @@ # /usr/share/logstash/pipeline.custom/1234_input_custom.conf ## # All of the defaults are loaded. +# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest. /usr/share/logstash/pipeline.dynamic/0900_input_redis.conf /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf /usr/share/logstash/pipeline.so/1002_preprocess_json.conf -/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf /usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf /usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf @@ -20,44 +21,44 @@ /usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf /usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf -/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf -/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf -/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf -/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf -/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf -/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf -/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf -/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf -/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf -/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf -/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf -/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf -/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf -/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf -/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf -/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf -/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf -/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf -/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf -/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf -/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf -/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf -/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf -/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf -/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf -/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf -/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf -/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf -/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf -/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf -/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf -/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf -/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf -/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf /usr/share/logstash/pipeline.so/1998_test_data.conf /usr/share/logstash/pipeline.so/2000_network_flow.conf -/usr/share/logstash/pipeline.so/6000_bro.conf -/usr/share/logstash/pipeline.so/6001_bro_import.conf +#/usr/share/logstash/pipeline.so/6000_bro.conf +#/usr/share/logstash/pipeline.so/6001_bro_import.conf /usr/share/logstash/pipeline.so/6002_syslog.conf /usr/share/logstash/pipeline.so/6101_switch_brocade.conf /usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf @@ -70,7 +71,7 @@ /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf /usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf /usr/share/logstash/pipeline.so/6700_winlogbeat.conf -/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf #/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf #/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf diff --git a/salt/logstash/etc/logstash-template.json b/salt/logstash/etc/logstash-template.json index 53c418d39..44e519842 100644 --- a/salt/logstash/etc/logstash-template.json +++ b/salt/logstash/etc/logstash-template.json @@ -2675,6 +2675,14 @@ "rule_number":{ "type":"long" }, + "rule_signature":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword" + } + } + }, "rule_type":{ "type":"text", "fields":{ diff --git a/salt/logstash/files/dynamic/9000_output_bro.conf b/salt/logstash/files/dynamic/9000_output_bro.conf index 54d2d20ca..553500281 100644 --- a/salt/logstash/files/dynamic/9000_output_bro.conf +++ b/salt/logstash/files/dynamic/9000_output_bro.conf @@ -20,6 +20,7 @@ output { if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] { # stdout { codec => rubydebug } elasticsearch { + pipeline => "%{event_type}" hosts => "{{ ES }}" index => "logstash-bro-%{+YYYY.MM.dd}" template_name => "logstash" diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 302598a45..589d22d1c 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -148,13 +148,13 @@ lslogdir: # Add the container so-logstashimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-logstash:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-logstash:HH1.1.1 so-logstash: docker_container.running: - require: - so-logstashimage - - image: soshybridhunter/so-logstash:HH1.1.0 + - image: soshybridhunter/so-logstash:HH1.1.1 - hostname: so-logstash - name: so-logstash - user: logstash @@ -193,3 +193,7 @@ so-logstash: {%- endif %} - watch: - file: /opt/so/conf/logstash/etc + - file: /opt/so/conf/logstash/conf.enabled.txt + - file: /opt/so/conf/logstash/custom + #- file: /opt/so/conf/logstash/rulesets + - file: /opt/so/conf/logstash/dynamic diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 5a67a6ec6..86408c0e3 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -96,13 +96,13 @@ stenolog: so-stenoimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-steno:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-steno:HH1.1.1 so-steno: docker_container.running: - require: - so-stenoimage - - image: soshybridhunter/so-steno:HH1.1.0 + - image: soshybridhunter/so-steno:HH1.1.1 - network_mode: host - privileged: True - port_bindings: @@ -116,6 +116,6 @@ so-steno: - /nsm/pcapout:/nsm/pcapout:rw - /opt/so/log/stenographer:/var/log/stenographer:rw - /opt/so/conf/steno/sensoroni.json:/opt/sensoroni/sensoroni.json:ro - - /opt/so/log/stenographer:/opt/sensoroni/log:rw + - /opt/so/log/stenographer:/opt/sensoroni/logs:rw - watch: - /opt/so/conf/steno/sensoroni.json diff --git a/salt/playbook/files/nav_layer_playbook.json b/salt/playbook/files/nav_layer_playbook.json new file mode 100644 index 000000000..43ed462b7 --- /dev/null +++ b/salt/playbook/files/nav_layer_playbook.json @@ -0,0 +1,28 @@ +{ + "name": "Playbook", + "version": "2.1", + "domain": "mitre-enterprise", + "description": "Current Coverage of Playbook", + "filters": { + "stages": ["act"], + "platforms": [ + "windows", + "linux", + "mac" + ] + }, + "sorting": 0, + "viewMode": 0, + "hideDisabled": "false", + "techniques": [{ + }], + "gradient": { + "colors": ["#ff6666", "#ffe766", "#8ec843"], + "minValue": 0, + "maxValue": 100 + }, + "metadata": [], + "showTacticRowBackground": "false", + "tacticRowBackground": "#dddddd", + "selectTechniquesAcrossTactics": "true" +} diff --git a/salt/playbook/files/navigator_config.json b/salt/playbook/files/navigator_config.json new file mode 100644 index 000000000..7e132cbf8 --- /dev/null +++ b/salt/playbook/files/navigator_config.json @@ -0,0 +1,59 @@ +{%- set ip = salt['pillar.get']('static:masterip', '') %} + +{ + "enterprise_attack_url": "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json", + "pre_attack_url": "https://raw.githubusercontent.com/mitre/cti/master/pre-attack/pre-attack.json", + "mobile_data_url": "https://raw.githubusercontent.com/mitre/cti/master/mobile-attack/mobile-attack.json", + "taxii_server": { + "enabled": false, + "url": "https://cti-taxii.mitre.org/", + "collections": { + "enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e", + "pre_attack": "062767bd-02d2-4b72-84ba-56caef0f8658", + "mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b" + } + }, + + "domain": "mitre-enterprise", + + "custom_context_menu_items": [ {"label": "view related plays","url": " https://{{ip}}/playbook/projects/playbook-prod/issues?utf8=%E2%9C%93&set_filter=1&sort=id%3Adesc&f%5B%5D=status_id&op%5Bstatus_id%5D=o&f%5B%5D=cf_27&op%5Bcf_27%5D=%3D&f%5B%5D=&c%5B%5D=status&c%5B%5D=cf_24&c%5B%5D=cf_25&c%5B%5D=cf_6&c%5B%5D=updated_on&group_by=&t%5B%5D=&v%5Bcf_27%5D%5B%5D=~Technique_ID~"}], + +"default_layers": { + "enabled": true, + "urls": [ + "assets/playbook.json" + ] + }, + + "comment_color": "yellow", + + "features": [ + {"name": "tabs", "enabled": true, "description": "Disable to remove the ability to open new tabs."}, + {"name": "selecting_techniques", "enabled": true, "description": "Disable to remove the ability to select techniques."}, + {"name": "header", "enabled": true, "description": "Disable to remove the header containing 'MITRE ATT&CK Navigator' and the link to the help page. The help page can still be accessed from the new tab menu."}, + {"name": "selection_controls", "enabled": true, "description": "Disable to to disable all subfeatures", "subfeatures": [ + {"name": "search", "enabled": true, "description": "Disable to remove the technique search panel from the interface."}, + {"name": "multiselect", "enabled": true, "description": "Disable to remove the multiselect panel from interface."}, + {"name": "deselect_all", "enabled": true, "description": "Disable to remove the deselect all button from the interface."} + ]}, + {"name": "layer_controls", "enabled": true, "description": "Disable to to disable all subfeatures", "subfeatures": [ + {"name": "layer_info", "enabled": true, "description": "Disable to remove the layer info (name, description and metadata) panel from the interface. Note that the layer can still be renamed in the tab."}, + {"name": "download_layer", "enabled": true, "description": "Disable to remove the button to download the layer."}, + {"name": "export_render", "enabled": true, "description": "Disable to the remove the button to render the current layer."}, + {"name": "export_excel", "enabled": true, "description": "Disable to the remove the button to export the current layer to MS Excel (.xlsx) format."}, + {"name": "filters", "enabled": true, "description": "Disable to the remove the filters panel from interface."}, + {"name": "sorting", "enabled": true, "description": "Disable to the remove the sorting button from the interface."}, + {"name": "color_setup", "enabled": true, "description": "Disable to the remove the color setup panel from interface, containing customization controls for scoring gradient and tactic row color."}, + {"name": "toggle_hide_disabled", "enabled": true, "description": "Disable to the remove the hide disabled techniques button from the interface."}, + {"name": "toggle_view_mode", "enabled": true, "description": "Disable to the remove the toggle view mode button from interface."}, + {"name": "legend", "enabled": true, "description": "Disable to the remove the legend panel from the interface."} + ]}, + {"name": "technique_controls", "enabled": true, "description": "Disable to to disable all subfeatures", "subfeatures": [ + {"name": "disable_techniques", "enabled": true, "description": "Disable to the remove the ability to disable techniques."}, + {"name": "manual_color", "enabled": true, "description": "Disable to the remove the ability to assign manual colors to techniques."}, + {"name": "scoring", "enabled": true, "description": "Disable to the remove the ability to score techniques."}, + {"name": "comments", "enabled": true, "description": "Disable to the remove the ability to add comments to techniques."}, + {"name": "clear_annotations", "enabled": true, "description": "Disable to remove the button to clear all annotations on the selected techniques."} + ]} + ] +} diff --git a/salt/playbook/files/redmine.db b/salt/playbook/files/redmine.db new file mode 100644 index 000000000..fdf24eae4 Binary files /dev/null and b/salt/playbook/files/redmine.db differ diff --git a/salt/playbook/init.sls b/salt/playbook/init.sls new file mode 100644 index 000000000..a72514fe9 --- /dev/null +++ b/salt/playbook/init.sls @@ -0,0 +1,58 @@ +{% set MASTERIP = salt['pillar.get']('master:mainip', '') %} + +playbookdb: + file.managed: + - name: /opt/so/conf/playbook/redmine.db + - source: salt://playbook/files/redmine.db + - user: 999 + - group: 999 + - makedirs: True + - replace: False + +playbookwebhook: + module.run: + - name: sqlite3.modify + - db: /opt/so/conf/playbook/redmine.db + - sql: "update webhooks set url = 'http://{{MASTERIP}}:7000/playbook/webhook' where project_id = 1" + +navigatorconfig: + file.managed: + - name: /opt/so/conf/playbook/navigator_config.json + - source: salt://playbook/files/navigator_config.json + - user: 939 + - group: 939 + - makedirs: True + - template: jinja + +so-playbookimage: + cmd.run: + - name: docker pull --disable-content-trust=false soshybridhunter/so-playbook:HH1.1.1 + +so-playbook: + docker_container.running: + - require: + - so-playbookimage + - image: soshybridhunter/so-playbook:HH1.1.1 + - hostname: playbook + - name: so-playbook + - binds: + - /opt/so/conf/playbook/redmine.db:/usr/src/redmine/sqlite/redmine.db:rw + - port_bindings: + - 0.0.0.0:3200:3000 + +so-navigatorimage: + cmd.run: + - name: docker pull --disable-content-trust=false soshybridhunter/so-navigator:HH1.1.1 + +so-navigator: + docker_container.running: + - require: + - so-navigatorimage + - image: soshybridhunter/so-navigator:HH1.1.1 + - hostname: navigator + - name: so-navigator + - binds: + - /opt/so/conf/playbook/navigator_config.json:/nav-app/src/assets/config.json:ro + - /opt/so/conf/playbook/nav_layer_playbook.json:/nav-app/src/assets/playbook.json:ro + - port_bindings: + - 0.0.0.0:4200:4200 diff --git a/salt/sensoroni/init.sls b/salt/sensoroni/init.sls index 9c6cf1906..2c06ba7a8 100644 --- a/salt/sensoroni/init.sls +++ b/salt/sensoroni/init.sls @@ -29,13 +29,13 @@ sensoronisync: so-sensoroniimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-sensoroni:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-sensoroni:HH1.1.1 so-sensoroni: docker_container.running: - require: - so-sensoroniimage - - image: soshybridhunter/so-sensoroni:HH1.1.0 + - image: soshybridhunter/so-sensoroni:HH1.1.1 - hostname: sensoroni - name: so-sensoroni - binds: diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 1a48ad92f..f1d311602 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -45,5 +45,9 @@ rtir_creator = root slack_url = YOURSLACKWORKSPACE slack_webhook = YOURSLACKWEBHOOK +[playbook] +playbook_url = http://{{ip}}:3200/playbook +playbook_key = a4a34538782804adfcb8dfae96262514ad70c37c + [log] logfile = /tmp/soctopus.log diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template new file mode 100644 index 000000000..992db3fa9 --- /dev/null +++ b/salt/soctopus/files/templates/generic.template @@ -0,0 +1,41 @@ +{% set es = salt['pillar.get']('static:masterip', '') %} +{% set hivehost = salt['pillar.get']('static:masterip', '') %} +{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +es_host: {{es}} +es_port: 9200 +name: Alert-Name +type: frequency +index: "*:logstash-*" +num_events: 1 +timeframe: + minutes: 10 +buffer_time: + minutes: 10 +allow_buffer_time_overlap: true + +filter: +- query: + query_string: + query: 'select from test' + +alert: modules.so.thehive.TheHiveAlerter + +hive_connection: + hive_host: https://{{hivehost}}/thehive/ + hive_apikey: {{hivekey}} + +hive_proxies: + http: '' + https: '' + +hive_alert_config: + title: '{rule[name]}' + type: 'external' + source: 'SecurityOnion' + description: '`Data:` {match[message]}' + severity: 2 + tags: ['elastalert', 'SecurityOnion'] + tlp: 3 + status: 'New' + follow: True + caseTemplate: '5000' diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template new file mode 100644 index 000000000..1e85a3182 --- /dev/null +++ b/salt/soctopus/files/templates/osquery.template @@ -0,0 +1,47 @@ +{% set es = salt['pillar.get']('static:masterip', '') %} +{% set hivehost = salt['pillar.get']('static:masterip', '') %} +{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +es_host: {{es}} +es_port: 9200 +name: Alert-Name +type: frequency +index: "*:logstash-*" +num_events: 1 +timeframe: + minutes: 10 +buffer_time: + minutes: 10 +allow_buffer_time_overlap: true + +filter: +- query: + query_string: + query: 'select from test' + +alert: modules.so.thehive.TheHiveAlerter + +hive_connection: + hive_host: https://{{hivehost}}/thehive/ + hive_apikey: {{hivekey}} + +hive_proxies: + http: '' + https: '' + +hive_alert_config: + title: '{rule[name]} -- {match[osquery][hostname]} -- {match[osquery][name]}' + type: 'external' + source: 'SecurityOnion' + description: '`Hostname:` __{match[osquery][hostname]}__ `Live Query:`__[Pivot Link](https://{{es}}/fleet/queries/new?host_uuids={match[osquery][LiveQuery]})__ `Pack:` __{match[osquery][name]}__ `Data:` {match[osquery][columns]}' + severity: 2 + tags: ['elastalert', 'SecurityOnion'] + tlp: 3 + status: 'New' + follow: True + caseTemplate: '5000' + +hive_observable_data_mapping: + - ip: '{match[osquery][EndpointIP1]}' + - ip: '{match[osquery][EndpointIP2]}' + - other: '{match[osquery][hostIdentifier]}' + - other: '{match[osquery][hostname]}' diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 22b21eb8f..98a9a4158 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -13,18 +13,45 @@ soctopussync: - group: 939 - template: jinja +playbookrulesdir: + file.directory: + - name: /opt/so/rules/elastalert/playbook + - user: 939 + - group: 939 + - makedirs: True + +playbookrulessync: + file.recurse: + - name: /opt/so/rules/elastalert/playbook + - source: salt://soctopus/files/templates + - user: 939 + - group: 939 + - template: jinja + +navigatordefaultlayer: + file.managed: + - name: /opt/so/conf/playbook/nav_layer_playbook.json + - source: salt://playbook/files/nav_layer_playbook.json + - user: 939 + - group: 939 + - makedirs: True + - replace: False + - template: jinja + so-soctopusimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-soctopus:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-soctopus:HH1.1.1 so-soctopus: docker_container.running: - require: - so-soctopusimage - - image: soshybridhunter/so-soctopus:HH1.1.0 + - image: soshybridhunter/so-soctopus:HH1.1.1 - hostname: soctopus - name: so-soctopus - binds: - /opt/so/conf/soctopus/SOCtopus.conf:/SOCtopus/SOCtopus.conf:ro + - /opt/so/rules/elastalert/playbook:/etc/playbook-rules:rw + - /opt/so/conf/playbook/nav_layer_playbook.json:/etc/playbook/nav_layer_playbook.json:rw - port_bindings: - 0.0.0.0:7000:7000 diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 2739f4318..2c3b1aba8 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -72,13 +72,13 @@ suriconfigsync: so-suricataimage: cmd.run: - - name: docker pull --disable-content-trust=false soshybridhunter/so-suricata:HH1.1.0 + - name: docker pull --disable-content-trust=false soshybridhunter/so-suricata:HH1.1.1 so-suricata: docker_container.running: - require: - so-suricataimage - - image: soshybridhunter/so-suricata:HH1.1.0 + - image: soshybridhunter/so-suricata:HH1.1.1 - privileged: True - environment: - INTERFACE={{ interface }} diff --git a/salt/top.sls b/salt/top.sls index f3f8c0a73..cf5d47699 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -3,6 +3,7 @@ {%- set WAZUH = salt['pillar.get']('master:wazuh', '0') -%} {%- set GRAFANA = salt['pillar.get']('master:grafana', '0') -%} {%- set THEHIVE = salt['pillar.get']('master:thehive', '0') -%} +{%- set PLAYBOOK = salt['pillar.get']('master:playbook', '0') -%} base: 'G@role:so-sensor': - ca @@ -55,6 +56,10 @@ base: {%- if THEHIVE != 0 %} - hive {%- endif %} + {%- if PLAYBOOK != 0 %} + - playbook + {%- endif %} + 'G@role:so-master': @@ -87,7 +92,10 @@ base: {%- if THEHIVE != 0 %} - hive {%- endif %} - + {%- if PLAYBOOK != 0 %} + - playbook + {%- endif %} + # Storage node logic diff --git a/salt/utility/init.sls b/salt/utility/init.sls index 845da59c7..d4ff00b3b 100644 --- a/salt/utility/init.sls +++ b/salt/utility/init.sls @@ -5,6 +5,7 @@ crossclusterson: cmd.script: - shell: /bin/bash + - cwd: /opt/so - runas: socore - source: salt://utility/bin/crossthestreams.sh - template: jinja @@ -15,6 +16,7 @@ crossclusterson: fixsearch: cmd.script: - shell: /bin/bash + - cwd: /opt/so - runas: socore - source: salt://utility/bin/eval.sh - template: jinja diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 4ff937c08..b83240dfa 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -5,7 +5,7 @@ ossecgroup: group.present: - name: ossec - gid: 945 - + # Add ossecm user ossecm: user.present: @@ -13,6 +13,8 @@ ossecm: - gid: 945 - home: /opt/so/wazuh - createhome: False + - allow_uid_change: True + - allow_gid_change: True # Add ossecr user ossecr: @@ -21,6 +23,8 @@ ossecr: - gid: 945 - home: /opt/so/wazuh - createhome: False + - allow_uid_change: True + - allow_gid_change: True # Add ossec user ossec: @@ -29,6 +33,8 @@ ossec: - gid: 945 - home: /opt/so/wazuh - createhome: False + - allow_uid_change: True + - allow_gid_change: True # Add wazuh agent wazuhpkgs: diff --git a/so-setup-network.sh b/so-setup-network.sh index 24a1ddf5b..cd043375f 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -61,20 +61,36 @@ add_master_hostfile() { } add_socore_user_master() { - echo "Add socore on the master" >> $SETUPLOG 2>&1 + + echo "Add socore on the master" >>~/sosetup.log 2>&1 + # Add user "socore" to the master. This will be for things like accepting keys. if [ $OS == 'centos' ]; then local ADDUSER=adduser else local ADDUSER=useradd fi - # Add user "socore" to the master. This will be for things like accepting keys. groupadd --gid 939 socore $ADDUSER --uid 939 --gid 939 --home-dir /opt/so socore - # Prompt the user to set a password for the user - passwd socore + # Set the password for socore that we got during setup + echo socore:$COREPASS1 | chpasswd --crypt-method=SHA512 } +#add_socore_user_master() { +# echo "Add socore on the master" >> $SETUPLOG 2>&1 +# if [ $OS == 'centos' ]; then +# local ADDUSER=adduser +# else +# local ADDUSER=useradd +# fi +# # Add user "socore" to the master. This will be for things like accepting keys. +# groupadd --gid 939 socore +# $ADDUSER --uid 939 --gid 939 --home-dir /opt/so socore +# # Prompt the user to set a password for the user +# passwd socore + +#} + add_socore_user_notmaster() { echo "Add socore user on non master" >> $SETUPLOG 2>&1 # Add socore user to the non master system. Probably not a bad idea to make system user @@ -168,6 +184,33 @@ checkin_at_boot() { echo "startup_states: highstate" >> /etc/salt/minion } +check_hive_init_then_reboot() { + WAIT_STEP=0 + MAX_WAIT=100 + until [ -f /opt/so/state/thehive.txt ] ; do + WAIT_STEP=$(( ${WAIT_STEP} + 1 )) + echo "Waiting on the_hive to init...Attempt #$WAIT_STEP" + if [ ${WAIT_STEP} -gt ${MAX_WAIT} ]; then + echo "ERROR: We waited ${MAX_WAIT} seconds but the_hive is not working." + exit 5 + fi + sleep 1s; + done + docker stop so-thehive + docker rm so-thehive + shutdown -r now +} + +check_socore_pass() { + + if [ $COREPASS1 == $COREPASS2 ]; then + SCMATCH=yes + else + whiptail_passwords_dont_match + fi + +} + chown_salt_master() { echo "Chown the salt dirs on the master for socore" >> $SETUPLOG 2>&1 @@ -246,31 +289,39 @@ copy_ssh_key() { } -create_bond_nmcli() { +network_setup() { echo "Setting up Bond" >> $SETUPLOG 2>&1 # Set the MTU - if [ $NSMSETUP != 'ADVANCED' ]; then + if [ "$NSMSETUP" != 'ADVANCED' ]; then MTU=1500 fi -# Create the bond interface - nmcli con add ifname bond0 con-name "bond0" type bond mode 0 -- \ - ipv4.method disabled \ - ipv6.method link-local \ - ethernet.mtu $MTU \ - connection.autoconnect "yes" >> $SETUPLOG 2>&1 + # Create the bond interface + nmcli con add ifname bond0 con-name "bond0" type bond mode 0 -- \ + ipv4.method disabled \ + ipv6.method link-local \ + ethernet.mtu $MTU \ + connection.autoconnect "yes" >> $SETUPLOG 2>&1 - for BNIC in ${BNICS[@]}; do - # Strip the quotes from the NIC names - BONDNIC="$(echo -e "${BNIC}" | tr -d '"')" - # Create the slave interface and assign it to the bond - nmcli con add type ethernet ifname $BONDNIC con-name "bond0-slave-$BONDNIC" master bond0 -- \ - ethernet.mtu $MTU \ - connection.autoconnect "yes" >> $SETUPLOG 2>&1 - # Bring the slave interface up - nmcli con up bond0-slave-$BONDNIC >> $SETUPLOG 2>&1 + for BNIC in ${BNICS[@]}; do + # Strip the quotes from the NIC names + BONDNIC="$(echo -e "${BNIC}" | tr -d '"')" + # Turn off various offloading settings for the interface + for i in rx tx sg tso ufo gso gro lro; do + ethtool -K $BONDNIC $i off >> $SETUPLOG 2>&1 done + # Create the slave interface and assign it to the bond + nmcli con add type ethernet ifname $BONDNIC con-name "bond0-slave-$BONDNIC" master bond0 -- \ + ethernet.mtu $MTU \ + connection.autoconnect "yes" >> $SETUPLOG 2>&1 + # Bring the slave interface up + nmcli con up bond0-slave-$BONDNIC >> $SETUPLOG 2>&1 + done + # Replace the variable string in the network script + sed -i "s/\$MAININT/${MAININT}/g" ./install_scripts/disable-checksum-offload.sh >> $SETUPLOG 2>&1 + # Copy the checksum offload script to prevent issues with packet capture + cp ./install_scripts/disable-checksum-offload.sh /etc/NetworkManager/dispatcher.d/disable-checksum-offload.sh >> $SETUPLOG 2>&1 } detect_os() { @@ -458,11 +509,11 @@ install_master() { ls_heapsize() { # Determine LS Heap Size - if [ $TOTAL_MEM -ge 16000 ] ; then - LS_HEAP_SIZE="4192m" + if [ $TOTAL_MEM -ge 32000 ] ; then + LS_HEAP_SIZE="1000m" else - # Set a max of 1GB heap if you have less than 16GB RAM - LS_HEAP_SIZE="2g" + # If minimal RAM, then set minimal heap + LS_HEAP_SIZE="500m" fi } @@ -477,8 +528,8 @@ master_pillar() { echo " esheap: $ES_HEAP_SIZE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " esclustername: {{ grains.host }}" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls if [ $INSTALLTYPE == 'EVALMODE' ]; then - echo " freq: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " domainstats: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " freq: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " domainstats: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " ls_pipeline_batch_size: 125" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " ls_input_threads: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " ls_batch_count: 125" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls @@ -505,6 +556,7 @@ master_pillar() { echo " osquery: $OSQUERY" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " wazuh: $WAZUH" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " thehive: $THEHIVE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " playbook: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls } master_static() { @@ -902,6 +954,15 @@ sensor_pillar() { } +set_hostname() { + + hostnamectl set-hostname --static $HOSTNAME + echo "127.0.0.1 $HOSTNAME $HOSTNAME.localdomain localhost localhost.localdomain localhost4 localhost4.localdomain" > /etc/hosts + echo "::1 localhost localhost.localdomain localhost6 localhost6.localdomain6" >> /etc/hosts + echo $HOSTNAME > /etc/hostname + +} + set_initial_firewall_policy() { get_main_ip @@ -1082,6 +1143,26 @@ whiptail_check_exitstatus() { } +whiptail_create_socore_user() { + + whiptail --title "Security Onion Setup" --msgbox "Set a password for the socore user. This account is used for adding sensors remotely." 8 78 + +} + +whiptail_create_socore_user_password1() { + + COREPASS1=$(whiptail --title "Security Onion Install" --passwordbox \ + "Enter a password for user socore" 10 60 3>&1 1>&2 2>&3) + +} + +whiptail_create_socore_user_password2() { + + COREPASS2=$(whiptail --title "Security Onion Install" --passwordbox \ + "Re-enter a password for user socore" 10 60 3>&1 1>&2 2>&3) + +} + whiptail_cur_close_days() { CURCLOSEDAYS=$(whiptail --title "Security Onion Setup" --inputbox \ @@ -1368,6 +1449,12 @@ whiptail_node_ls_input_batch_count() { } +whiptail_passwords_dont_match() { + + whiptail --title "Security Onion Setup" --msgbox "Passwords don't match. Please re-enter." 8 78 + +} + whiptail_rule_setup() { # Get pulled pork info @@ -1395,19 +1482,27 @@ whiptail_sensor_config() { } +whiptail_set_hostname() { + + HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \ + "Enter the Hostname you would like to set." 10 60 localhost 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + whiptail_setup_complete() { - whiptail --title "Security Onion Setup" --msgbox "Finished installing this as an $INSTALLTYPE." 8 78 + whiptail --title "Security Onion Setup" --msgbox "Finished installing this as an $INSTALLTYPE. Press Enter to reboot." 8 78 install_cleanup - exit } whiptail_setup_failed() { - whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $SETUPLOG for details" 8 78 + whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $SETUPLOG for details. Press Enter to reboot." 8 78 install_cleanup - exit } @@ -1491,6 +1586,9 @@ if (whiptail_you_sure); then # Let folks know they need their management interface already set up. whiptail_network_notice + # Set the hostname to reduce errors + whiptail_set_hostname + # Go ahead and gen the keys so we can use them for any sensor type - Disabled for now #minio_generate_keys @@ -1540,8 +1638,17 @@ if (whiptail_you_sure); then fi fi + whiptail_create_socore_user + SCMATCH=no + while [ $SCMATCH != yes ]; do + whiptail_create_socore_user_password1 + whiptail_create_socore_user_password2 + check_socore_pass + done + # Last Chance to back out whiptail_make_changes + set_hostname generate_passwords auth_pillar clear_master @@ -1555,9 +1662,9 @@ if (whiptail_you_sure); then get_main_ip # Add the user so we can sit back and relax - echo "" - echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors" - echo "" + #echo "" + #echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors" + #echo "" add_socore_user_master # Install salt and dependencies @@ -1643,8 +1750,14 @@ if (whiptail_you_sure); then GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}') if [[ $GOODSETUP == '0' ]]; then whiptail_setup_complete + if [[ $THEHIVE == '1' ]]; then + check_hive_init_then_reboot + else + shutdown -r now + fi else whiptail_setup_failed + shutdown -r now fi fi @@ -1673,6 +1786,7 @@ if (whiptail_you_sure); then whiptail_basic_suri fi whiptail_make_changes + set_hostname clear_master mkdir -p /nsm get_filesystem_root @@ -1683,7 +1797,7 @@ if (whiptail_you_sure); then echo -e "XXX\n0\nSetting Initial Firewall Policy... \nXXX" set_initial_firewall_policy >> $SETUPLOG 2>&1 echo -e "XXX\n3\nCreating Bond Interface... \nXXX" - create_bond_nmcli >> $SETUPLOG 2>&1 + network_setup >> $SETUPLOG 2>&1 echo -e "XXX\n4\nGenerating Sensor Pillar... \nXXX" sensor_pillar >> $SETUPLOG 2>&1 echo -e "XXX\n5\nInstalling Salt Components... \nXXX" @@ -1716,8 +1830,10 @@ if (whiptail_you_sure); then GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}') if [[ $GOODSETUP == '0' ]]; then whiptail_setup_complete + shutdown -r now else whiptail_setup_failed + shutdown -r now fi fi @@ -1744,7 +1860,7 @@ if (whiptail_you_sure); then es_heapsize ls_heapsize NODE_ES_HEAP_SIZE="600m" - NODE_LS_HEAP_SIZE="2000m" + NODE_LS_HEAP_SIZE="500m" LSPIPELINEWORKERS=1 LSPIPELINEBATCH=125 LSINPUTTHREADS=1 @@ -1755,8 +1871,15 @@ if (whiptail_you_sure); then BROVERSION=ZEEK CURCLOSEDAYS=30 process_components + whiptail_create_socore_user + SCMATCH=no + while [ $SCMATCH != yes ]; do + whiptail_create_socore_user_password1 + whiptail_create_socore_user_password2 + check_socore_pass + done whiptail_make_changes - #eval_mode_hostsfile + set_hostname generate_passwords auth_pillar clear_master @@ -1766,14 +1889,11 @@ if (whiptail_you_sure); then get_log_size_limit get_main_ip # Add the user so we can sit back and relax - echo "" - echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors" - echo "" add_socore_user_master { sleep 0.5 echo -e "XXX\n0\nCreating Bond Interface... \nXXX" - create_bond_nmcli >> $SETUPLOG 2>&1 + network_setup >> $SETUPLOG 2>&1 echo -e "XXX\n1\nInstalling saltstack... \nXXX" saltify >> $SETUPLOG 2>&1 echo -e "XXX\n3\nInstalling docker... \nXXX" @@ -1849,6 +1969,7 @@ if (whiptail_you_sure); then salt-call state.apply schedule >> $SETUPLOG 2>&1 salt-call state.apply soctopus >> $SETUPLOG 2>&1 if [[ $THEHIVE == '1' ]]; then + echo -e "XXX\n97\nInstalling The Hive... \nXXX" salt-call state.apply hive >> $SETUPLOG 2>&1 fi echo -e "XXX\n98\nSetting checkin to run on boot... \nXXX" @@ -1861,14 +1982,26 @@ if (whiptail_you_sure); then if [ $OS == 'centos' ]; then if [[ $GOODSETUP == '1' ]]; then whiptail_setup_complete + if [[ $THEHIVE == '1' ]]; then + check_hive_init_then_reboot + else + shutdown -r now + fi else whiptail_setup_failed + shutdown -r now fi else if [[ $GOODSETUP == '0' ]]; then whiptail_setup_complete + if [[ $THEHIVE == '1' ]]; then + check_hive_init_then_reboot + else + shutdown -r now + fi else whiptail_setup_failed + shutdown -r now fi fi fi @@ -1905,6 +2038,7 @@ if (whiptail_you_sure); then LSINPUTBATCHCOUNT=125 fi whiptail_make_changes + set_hostname clear_master mkdir -p /nsm get_filesystem_root @@ -1946,26 +2080,28 @@ if (whiptail_you_sure); then GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}') if [[ $GOODSETUP == '0' ]]; then whiptail_setup_complete + shutdown -r now else whiptail_setup_failed + shutdown -r now fi - set_initial_firewall_policy - saltify - docker_install - configure_minion node - set_node_type - node_pillar - copy_minion_pillar nodes - salt_checkin + #set_initial_firewall_policy + #saltify + #docker_install + #configure_minion node + #set_node_type + #node_pillar + #copy_minion_pillar nodes + #salt_checkin # Accept the Salt Key - accept_salt_key_remote + #accept_salt_key_remote # Do the big checkin but first let them know it will take a bit. - salt_checkin_message - salt_checkin - checkin_at_boot + #salt_checkin_message + #salt_checkin + #checkin_at_boot - whiptail_setup_complete + #whiptail_setup_complete fi else