Merge pull request #70 from TOoSmOotH/master

Alpha 1.1.1

README.md

@@ -2,19 +2,20 @@
 
 ### Changes:
 
-- Alpha 2 is here!! Check out the [Hybrid Hunter Quick Start Guide](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/Hybrid-Hunter-Quick-Start-Guide).  
-- Suricata 4.1.5  
-- Bro/Zeek 2.6.4  
-- TheHive 3.4.0 (ES to 6.8.3)
-- NIDS and HIDS dashboard updates
+- Alpha 2 is here!
+- Suricata 4.1.5.  
+- Bro/Zeek 2.6.4.  
+- TheHive 3.4.0 (Includes ES 6.8.3 for TheHive only).
+- Fixed Bro/Zeek packet loss calculation for Grafana.
+- Updated to latest Sensoroni which includes websockets support for job status updates without having to refresh the page.
+- NIDS and HIDS dashboard updates.
 - Playbook and ATT&CK Navigator features are now included.
 - Filebeat now logs to a file, instead of stdout.
 - Elastalert has been updated to use Python 3 and allow for use of custom alerters.  
-- Elasticsearch Ingest is now used to consume Zeek logs and Suricata alerts (instead of the traditional Logstash pipeline).
-  This reduces the memory footprint of Logstash dramatically!  
+- Moved Bro/Zeek log parsing from Logstash to Elasticsearch Ingest for higher performance and lower memory usage!
 - Several changes to the setup script have been made to improve stability of the setup process:  
-  - Setup now modifies your hosts file so that the install works better in environments without DNS  
-  - You are now prompted for setting a password for the socore user  
+  - Setup now modifies your hosts file so that the install works better in environments without DNS.  
+  - You are now prompted for setting a password for the socore user.  
   - The install now forces a reboot at the end of the install. This fixes an issue with some of the Docker containers being in the wrong state from a manual reboot. Manual reboots are fine after the initial reboot.
 
 
@@ -47,22 +48,9 @@ Distributed:
 
 ### Prerequisites
 
-If you are running CentOS 7 there are a couple of prerequisites:
-
-```
-sudo yum -y install git bind-utils
-sudo hostnamectl set-hostname YOURHOSTNAME
-sudo reboot
-```
-
-If you are running CentOS 7 or Ubuntu 16.04 and don't have name resolution ensure your `/etc/hosts` file looks like this:
-
-```
-127.0.0.1   YOURHOSTNAME YOURHOSTNAME.localdomain localhost localhost.localdomain localhost4 localhost4.localdomain4
-::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
-```  
-It is imperative that YOURHOSTNAME.localdomain is included in this hosts entry for the install to complete properly.
+Install git if using a Centos 7 Minimal install:
 
+```sudo yum -y install git```
 
 ### Installation
 

install_scripts/disable-checksum-offload.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ "$NM_DISPATCHER_ACTION" == "pre-up" ]; then
+    if ["$DEVICE_IFACE" !== "$MAININT"]; then
+        for i in rx tx sg tso ufo gso gro lro; do 
+            ethtool -K $DEVICE_IFACE $i off; 
+        done
+    fi
+fi

salt/bro/cron/packetloss.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-/usr/bin/docker exec so-bro /opt/bro/bin/broctl netstats | awk -F '[ =]' '{RCVD += $5;DRP += $7;TTL += $9} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/bro/logs/packetloss.log
+/usr/bin/docker exec so-bro /opt/bro/bin/broctl netstats | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/bro/logs/packetloss.log

salt/bro/init.sls

@@ -125,13 +125,13 @@ localbrosync:
 
 so-broimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-bro:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-bro:HH1.1.1
 
 so-bro:
   docker_container.running:
     - require:
       - so-broimage
-    - image: soshybridhunter/so-bro:HH1.1.0
+    - image: soshybridhunter/so-bro:HH1.1.1
     - privileged: True
     - binds:
       - /nsm/bro/logs:/nsm/bro/logs:rw

salt/common/init.sls

@@ -116,13 +116,13 @@ nginxtmp:
 # Start the core docker
 so-coreimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-core:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-core:HH1.1.1
 
 so-core:
   docker_container.running:
     - require:
       - so-coreimage
-    - image: soshybridhunter/so-core:HH1.1.0
+    - image: soshybridhunter/so-core:HH1.1.1
     - hostname: so-core
     - user: socore
     - binds:

salt/common/nginx/nginx.conf.so-eval

@@ -113,6 +113,31 @@ http {
 
         }
 
+        location /playbook/ {
+          proxy_pass http://{{ masterip }}:3200/playbook/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
+
+
+        location /navigator/ {
+          auth_basic            "Security Onion";
+          auth_basic_user_file  /opt/so/conf/nginx/.htpasswd;
+          proxy_pass http://{{ masterip }}:4200/navigator/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
+
         location /api/ {
           proxy_pass https://{{ masterip }}:8080/api/;
           proxy_read_timeout    90;
@@ -173,6 +198,8 @@ http {
           proxy_set_header      X-Real-IP $remote_addr;
           proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
           proxy_set_header      Proxy "";
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "Upgrade";
 
         }
 

salt/common/nginx/nginx.conf.so-master

@@ -113,6 +113,30 @@ http {
 
         }
 
+       location /playbook/ {
+          proxy_pass http://{{ masterip }}:3200/playbook/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
+
+        location /navigator/ {
+          auth_basic            "Security Onion";
+          auth_basic_user_file  /opt/so/conf/nginx/.htpasswd;
+          proxy_pass http://{{ masterip }}:4200/navigator/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
+
         location /api/ {
           proxy_pass https://{{ masterip }}:8080/api/;
           proxy_read_timeout    90;
@@ -173,6 +197,8 @@ http {
           proxy_set_header      X-Real-IP $remote_addr;
           proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
           proxy_set_header      Proxy "";
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "Upgrade";
 
         }
 

salt/elastalert/files/elastalert_config.yaml

@@ -0,0 +1,79 @@
+{% set esip = salt['pillar.get']('master:mainip', '') %}
+{% set esport = salt['pillar.get']('master:es_port', '') %}
+# This is the folder that contains the rule yaml files
+# Any .yaml file will be loaded as a rule
+rules_folder: /etc/elastalert/rules/
+
+# Sets whether or not ElastAlert should recursively descend
+# the rules directory - true or false
+scan_subdirectories: true
+
+# How often ElastAlert will query Elasticsearch
+# The unit can be anything from weeks to seconds
+run_every:
+  minutes: 1
+
+# ElastAlert will buffer results from the most recent
+# period of time, in case some log sources are not in real time
+buffer_time:
+  minutes: 1
+
+# The maximum time between queries for ElastAlert to start at the most recently
+# run query. When ElastAlert starts, for each rule, it will search elastalert_metadata
+# for the most recently run query and start from that time, unless it is older than
+# old_query_limit, in which case it will start from the present time. The default is one week.
+old_query_limit:
+  minutes: 5
+
+# The Elasticsearch hostname for metadata writeback
+# Note that every rule can have its own Elasticsearch host
+es_host: {{ esip }}
+
+# The Elasticsearch port
+es_port: {{ esport }}
+
+# Sets timeout for connecting to and reading from es_host
+es_conn_timeout: 60
+
+# The maximum number of documents that will be downloaded from Elasticsearch in
+# a single query. The default is 10,000, and if you expect to get near this number,
+# consider using use_count_query for the rule. If this limit is reached, ElastAlert
+# will scroll through pages the size of max_query_size until processing all results.
+max_query_size: 5000
+
+# The AWS region to use. Set this when using AWS-managed elasticsearch
+#aws_region: us-east-1
+
+# The AWS profile to use. Use this if you are using an aws-cli profile.
+# See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+# for details
+#profile: test
+
+# Optional URL prefix for Elasticsearch
+#es_url_prefix: elasticsearch
+
+# Connect with TLS to Elasticsearch
+#use_ssl: True
+
+# Verify TLS certificates
+#verify_certs: True
+
+# GET request with body is the default option for Elasticsearch.
+# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
+# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
+# for details
+#es_send_get_body_as: GET
+
+# Option basic-auth username and password for Elasticsearch
+#es_username: someusername
+#es_password: somepassword
+
+# The index on es_host which is used for metadata storage
+# This can be a unmapped index, but it is recommended that you run
+# elastalert-create-index to set a mapping
+writeback_index: elastalert_status
+
+# If an alert fails for some reason, ElastAlert will retry
+# sending the alert until this time period has elapsed
+alert_time_limit:
+  days: 2

salt/elastalert/files/modules/so/thehive.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import uuid
+import re
+
+from elastalert.alerts import Alerter
+from thehive4py.api import TheHiveApi
+from thehive4py.models import Alert, AlertArtifact, CustomFieldHelper
+
+class TheHiveAlerter(Alerter):
+    """
+    Use matched data to create alerts containing observables in an instance of TheHive
+    This is a modified version for use with Security Onion
+    """
+
+    required_options = set(['hive_connection', 'hive_alert_config'])
+
+    def alert(self, matches):
+
+        connection_details = self.rule['hive_connection']
+
+        api = TheHiveApi(
+            connection_details.get('hive_host'),
+            connection_details.get('hive_apikey', ''),
+            proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}),
+            cert=connection_details.get('hive_verify', False))
+
+        for match in matches:
+            context = {'rule': self.rule, 'match': match}
+
+            artifacts = []
+            for mapping in self.rule.get('hive_observable_data_mapping', []):
+                for observable_type, match_data_key in mapping.items():
+                    try:
+                        match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key)
+                        rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key)
+                        data_keys = match_data_keys + rule_data_keys
+                        context_keys = list(context['match'].keys()) + list(context['rule'].keys())
+                        if all([True if k in context_keys else False for k in data_keys]):
+                            artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context)))
+                    except KeyError:
+                        raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context))
+
+            alert_config = {
+                'artifacts': artifacts,
+                'sourceRef': str(uuid.uuid4())[0:6],
+                'title': '{rule[index]}_{rule[name]}'.format(**context)
+            }
+            alert_config.update(self.rule.get('hive_alert_config', {}))
+
+            for alert_config_field, alert_config_value in alert_config.items():
+                if alert_config_field == 'customFields':
+                    custom_fields = CustomFieldHelper()
+                    for cf_key, cf_value in alert_config_value.items():
+                        try:
+                            func = getattr(custom_fields, 'add_{}'.format(cf_value['type']))
+                        except AttributeError:
+                            raise Exception('unsupported custom field type {}'.format(cf_value['type']))
+                        value = cf_value['value'].format(**context)
+                        func(cf_key, value)
+                    alert_config[alert_config_field] = custom_fields.build()
+                elif isinstance(alert_config_value, str):
+                    alert_config[alert_config_field] = alert_config_value.format(**context)
+                elif isinstance(alert_config_value, (list, tuple)):
+                    formatted_list = []
+                    for element in alert_config_value:
+                        try:
+                            formatted_list.append(element.format(**context))
+                        except (AttributeError, KeyError, IndexError):
+                            formatted_list.append(element)
+                    alert_config[alert_config_field] = formatted_list
+
+            alert = Alert(**alert_config)
+            response = api.create_alert(alert)
+
+            if response.status_code != 201:
+                raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
+
+    def get_info(self):
+
+        return {
+            'type': 'hivealerter',
+            'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '')
+       } 

salt/elastalert/files/rules/so/nids2hive.yaml

@@ -0,0 +1,50 @@
+{% set es = salt['pillar.get']('static:masterip', '') %}
+{% set hivehost = salt['pillar.get']('static:masterip', '') %}
+{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
+# hive.yaml
+# Elastalert rule to forward IDS alerts from Security Onion to a specified TheHive instance.
+#
+es_host: {{es}}
+es_port: 9200
+name: NIDS-Alert
+type: frequency
+index: "*:logstash-ids*"
+num_events: 1
+timeframe:
+    minutes: 10
+buffer_time:
+    minutes: 10
+allow_buffer_time_overlap: true
+query_key: alert
+realert:
+    days: 1
+
+filter:
+- query:
+   query_string:
+      query: "event_type: ids AND NOT tags: _jsonparsefailure"
+
+alert: modules.so.thehive.TheHiveAlerter
+
+hive_connection:
+  hive_host: https://{{hivehost}}/thehive/
+  hive_apikey: {{hivekey}}
+
+hive_proxies:
+  http: ''
+  https: ''
+
+hive_alert_config:
+  title: '{match[alert]}'
+  type: 'external'
+  source: 'SecurityOnion'
+  description: "`NIDS Dashboard:` \n\n <https://{{es}}/kibana/app/kibana#/dashboard/ed6f7e20-e060-11e9-8f0c-2ddbf5ed9290?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'sid:{match[sid]}')),sort:!('@timestamp',desc))> \n\n `IPs: `{match[source_ip]}:{match[source_port]} --> {match[destination_ip]}:{match[destination_port]} \n\n `Signature:` {match[rule_signature]}"
+  severity: 2
+  tags: ['elastalert', 'SecurityOnion', 'NIDS']
+  tlp: 3
+  status: 'New'
+  follow: True
+
+hive_observable_data_mapping:
+  - ip: '{match[source_ip]}'
+  - ip: '{match[destination_ip]}'

salt/elastalert/init.sls

@@ -64,38 +64,69 @@ elastarules:
     - group: 939
     - makedirs: True
 
-#elastaconfdir:
-#  file.directory:
-#    - name: /opt/so/conf/elastalert
-#    - user: 933
-#    - group: 939
-#    - makedirs: True
+elastaconfdir:
+  file.directory:
+    - name: /opt/so/conf/elastalert
+    - user: 933
+    - group: 939
+    - makedirs: True
 
-#elastaconf:
-#  file.managed:
-#    - name: /opt/so/conf/elastalert/config.yaml
-#    - source: salt://elastalert/files/config.yaml
-#    - user: 933
-#    - group: 939
-#    - template: jinja
+elastasomodulesdir:
+  file.directory:
+    - name: /opt/so/conf/elastalert/modules/so
+    - user: 933
+    - group: 939
+    - makedirs: True
+
+elastacustmodulesdir:
+  file.directory:
+    - name: /opt/so/conf/elastalert/modules/custom
+    - user: 933
+    - group: 939
+    - makedirs: True
+
+elastasomodulesync:
+  file.recurse:
+    - name: /opt/so/conf/elastalert/modules/so
+    - source: salt://elastalert/files/modules/so
+    - user: 933
+    - group: 939
+    - makedirs: True
+
+elastarulesync:
+  file.recurse:
+    - name: /opt/so/rules/elastalert
+    - source: salt://elastalert/files/rules/so
+    - user: 933
+    - group: 939
+    - template: jinja
+
+elastaconf:
+  file.managed:
+    - name: /opt/so/conf/elastalert/elastalert_config.yaml
+    - source: salt://elastalert/files/elastalert_config.yaml
+    - user: 933
+    - group: 939
+    - template: jinja
 
 so-elastalertimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-elastalert:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-elastalert:HH1.1.1
 
 so-elastalert:
   docker_container.running:
     - require:
       - so-elastalertimage
-    - image: soshybridhunter/so-elastalert:HH1.1.0
+    - image: soshybridhunter/so-elastalert:HH1.1.1
     - hostname: elastalert
     - name: so-elastalert
     - user: elastalert
     - detach: True
     - binds:
-#      - /opt/so/conf/elastalert/config.yaml:/etc/elastalert/conf/elastalert_config.yaml:ro
       - /opt/so/rules/elastalert:/etc/elastalert/rules/:ro
       - /opt/so/log/elastalert:/var/log/elastalert:rw
+      - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
+      - /opt/so/conf/elastalert/elastalert_config.yaml:/etc/elastalert/conf/elastalert_config.yaml:ro
     - environment:
       - ELASTICSEARCH_HOST: {{ esip }}
       - ELASTICSEARCH_PORT: {{ esport }}

salt/elasticsearch/files/ingest/bro_common

@@ -0,0 +1,9 @@
+{
+  "description" : "bro_common",
+  "processors" : [
+    { "rename":		{ "field": "@timestamp",	"target_field": "timestamp",	"ignore_missing": true					} },
+    { "date":		{ "field": "message2.ts",	"target_field": "@timestamp",	"formats": ["ISO8601", "UNIX"], "ignore_failure": true	} },
+    { "remove":		{ "field": "message2.ts",	"ignore_failure": true									} },
+    { "pipeline":	{ "name": "common" 													} }
+  ]
+}

salt/elasticsearch/files/ingest/bro_common_ssl

@@ -0,0 +1,58 @@
+{
+  "description" : "bro_common_ssl",
+  "processors" : [
+    {
+  	"kv": {
+    		"field": "certificate_issuer",
+		"field_split": ",",
+		"value_split": "=",
+		"ignore_missing": true,
+		"ignore_failure": true,
+		"include_keys": [ "CN", "C", "O", "OU", "ST", "SN", "L", "DC", "GN", "pseudonym", "serialNumber", "title", "initials" ]
+	}
+    },
+    { 	"rename":{ "field": "CN",		"target_field": "issuer_common_name",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "C",		"target_field": "issuer_country_code",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "O",		"target_field": "issuer_organization",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "OU",		"target_field": "issuer_organization_unit",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "ST",		"target_field": "issuer_state",			"ignore_failure": true 	} },
+    { 	"rename":{ "field": "SN",		"target_field": "issuer_surname",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "L",		"target_field": "issuer_locality",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "DC",		"target_field": "issuer_distinguised_name",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "GN",		"target_field": "issuer_given_name",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "pseudonym",	"target_field": "issuer_pseudonym",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "serialNumber",	"target_field": "issuer_serial_number",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "title",		"target_field": "issuer_title",			"ignore_failure": true 	} },
+    { 	"rename":{ "field": "initials",		"target_field": "issuer_initials",		"ignore_failure": true 	} },
+    {
+  	"kv": {
+    		"field": "certificate_subject",
+		"field_split": ",",
+		"value_split": "=",
+		"ignore_missing": true,
+		"ignore_failure": true,
+		"include_keys": [ "CN", "C", "O", "OU", "ST", "SN", "L", "GN", "pseudonym", "serialNumber", "title", "initials" ]
+	}
+    },
+    { 	"rename":{ "field": "CN",		"target_field": "certificate_common_name",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "C",		"target_field": "certificate_country_code",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "O",		"target_field": "certificate_organization",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "OU",		"target_field": "certificate_organization_unit","ignore_failure": true 	} },
+    { 	"rename":{ "field": "ST",		"target_field": "certificate_state",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "SN",		"target_field": "certificate_surname",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "L",		"target_field": "certificate_locality",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "GN",		"target_field": "certificate_given_name",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "pseudonym",	"target_field": "certificate_pseudonym",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "serialNumber",	"target_field": "certificate_serial_number",	"ignore_failure": true 	} },
+    { 	"rename":{ "field": "title",		"target_field": "certificate_title",		"ignore_failure": true 	} },
+    { 	"rename":{ "field": "initials",		"target_field": "certificate_initials",		"ignore_failure": true 	} },
+    { 	"script":{ "lang": "painless", "source": "ctx.certificate_common_name_length = ctx.certificate_common_name.length()", "ignore_failure": true } },
+    { 	"script":{ "lang": "painless", "source": "ctx.issuer_common_name_length = ctx.issuer_common_name.length()", "ignore_failure": true } },
+    { 	"script":{ "lang": "painless", "source": "ctx.server_name_length = ctx.server_name.length()", "ignore_failure": true } },
+    { 
+	"pipeline": { 
+		"name": "bro_common"
+	}
+    }
+  ]
+}

salt/elasticsearch/files/ingest/bro_conn

@@ -0,0 +1,48 @@
+{
+  "description" : "bro_conn",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.service", 		"target_field": "service",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.duration", 	"target_field": "duration",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_bytes", 	"target_field": "original_bytes",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_bytes", 	"target_field": "respond_bytes",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.conn_state", 	"target_field": "connection_state",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.local_orig", 	"target_field": "local_orig",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.local_resp", 	"target_field": "local_respond",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.missed_bytes", 	"target_field": "missed_bytes",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.history", 		"target_field": "history",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_pkts", 	"target_field": "original_packets",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_ip_bytes", 	"target_field": "original_ip_bytes",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_pkts", 	"target_field": "respond_packets",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_ip_bytes", 	"target_field": "respond_ip_bytes",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tunnel_parents",	"target_field": "tunnel_parents",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_cc", 		"target_field": "original_country_code","ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_cc", 		"target_field": "respond_country_code",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sensorname", 	"target_field": "sensor_name",		"ignore_missing": true 	} },
+    { "script":		{ "lang": "painless", "source":	"ctx.total_bytes = (ctx.original_bytes + ctx.respond_bytes)", "ignore_failure": true } },
+    { "set": { "if": "ctx.connection_state == 'S0'", 	"field": "connection_state_description", "value": "Connection attempt seen, no reply" } },
+    { "set": { "if": "ctx.connection_state == 'S1'", 	"field": "connection_state_description", "value": "Connection established, not terminated" } },
+    { "set": { "if": "ctx.connection_state == 'S2'", 	"field": "connection_state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } },
+    { "set": { "if": "ctx.connection_state == 'S3'", 	"field": "connection_state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } },
+    { "set": { "if": "ctx.connection_state == 'SF'", 	"field": "connection_state_description", "value": "Normal SYN/FIN completion" } },
+    { "set": { "if": "ctx.connection_state == 'REJ'", 	"field": "connection_state_description", "value": "Connection attempt rejected" } },
+    { "set": { "if": "ctx.connection_state == 'RSTO'", 	"field": "connection_state_description", "value": "Connection established, originator aborted (sent a RST)" } },
+    { "set": { "if": "ctx.connection_state == 'RSTR'", 	"field": "connection_state_description", "value": "Established, responder aborted" } },
+    { "set": { "if": "ctx.connection_state == 'RSTOS0'","field": "connection_state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } },
+    { "set": { "if": "ctx.connection_state == 'RSTRH'", "field": "connection_state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } },
+    { "set": { "if": "ctx.connection_state == 'SH'", 	"field": "connection_state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
+    { "set": { "if": "ctx.connection_state == 'SHR'", 	"field": "connection_state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
+    { "set": { "if": "ctx.connection_state == 'OTH'", 	"field": "connection_state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
+    { "pipeline": 	{ "name": "bro_common" } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_dce_rpc

@@ -0,0 +1,20 @@
+{
+  "description" : "bro_dce_rpc",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rtt", 		"target_field": "rtt",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.named_pipe",	"target_field": "named_pipe",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.endpoint", 	"target_field": "endpoint",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.operation", 	"target_field": "operation",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_dhcp

@@ -0,0 +1,20 @@
+{
+  "description" : "bro_dhcp",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uids", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mac", 		"target_field": "mac",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.assigned_ip",	"target_field": "assigned_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.lease_time", 	"target_field": "lease_time",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.trans_id", 	"target_field": "transaction_id",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.assigned_addr", 	"target_field": "assigned_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_addr", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_addr", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.requested_addr", 	"target_field": "requested_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.domain", 		"target_field": "domain_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host_name",	"target_field": "hostname",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.duration",		"target_field": "duration",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.msg_types", 	"target_field": "message_types",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_dnp3

@@ -0,0 +1,19 @@
+{
+  "description" : "bro_dnp3",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fc_request",	"target_field": "fc_request",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fc_reply", 	"target_field": "fc_reply",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.iin", 		"target_field": "iin",			"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_dns

@@ -0,0 +1,35 @@
+{
+  "description" : "bro_dns",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.trans_id",		"target_field": "transaction_id",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rtt",	 	"target_field": "rtt",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.query", 		"target_field": "query",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.qclass", 		"target_field": "query_class",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.qclass_name", 	"target_field": "query_class_name",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.qtype", 		"target_field": "query_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.qtype_name", 	"target_field": "query_type_name",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rcode",	 	"target_field": "rcode",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rcode_name", 	"target_field": "rcode_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.AA", 		"target_field": "aa",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.TC", 		"target_field": "tc",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.RD", 		"target_field": "rd",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.RA", 		"target_field": "ra",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Z",		"target_field": "z",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.answers", 		"target_field": "answers",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.TTLs", 		"target_field": "ttls",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rejected", 	"target_field": "rejected",		"ignore_missing": true 	} },
+    { "script": 	{ "lang": "painless", "source": "ctx.query_length = ctx.query.length()",	"ignore_failure": true  } },
+    { "pipeline":       { "name": "bro_common"											} }
+  ]
+}

salt/elasticsearch/files/ingest/bro_dpd

@@ -0,0 +1,19 @@
+{
+  "description" : "bro_dpd",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.analyzer", 	"target_field": "analyzer",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.failure_reason", 	"target_field": "failure_reason",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_files

@@ -0,0 +1,32 @@
+{
+  "description" : "bro_files",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.fuid", 	 	"target_field": "fuid",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tx_hosts", 	"target_field": "file_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rx_hosts.0",	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "remove": 	{ "field": "message2.rx_hosts",							"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.conn_uids", 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "remove": 	{ "field": "source",								"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.source", 		"target_field": "source",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.depth",	 	"target_field": "depth",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.analyzers", 	"target_field": "analyzer",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mime_type", 	"target_field": "mimetype",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.filename", 	"target_field": "file_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.duration", 	"target_field": "duration",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.local_orig",	"target_field": "local_orig",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.is_orig", 		"target_field": "is_orig",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.seen_bytes", 	"target_field": "seen_bytes",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.total_bytes", 	"target_field": "total_bytes",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.missing_bytes", 	"target_field": "missing_bytes",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.overflow_bytes",	"target_field": "overflow_bytes",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.timedout",		"target_field": "timed_out",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.parent_fuid",	"target_field": "parent_fuid",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.md5",		"target_field": "md5",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sha1",		"target_field": "sha1",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.extracted",	"target_field": "extracted",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.extracted_cutoff",	"target_field": "extracted_cutoff",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.extracted_size",	"target_field": "extracted_size",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_ftp

@@ -0,0 +1,33 @@
+{
+  "description" : "bro_http",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user",		"target_field": "username",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.password",		"target_field": "password",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.command", 		"target_field": "ftp_command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.arg", 		"target_field": "ftp_argument",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mime_type", 	"target_field": "mimetype",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.file_size", 	"target_field": "file_size",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.reply_code", 	"target_field": "reply_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.reply_msg", 	"target_field": "reply_message",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "data_channel.passive", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.data_channel.passive","target_field": "data_channel_passive",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "data_channel.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.data_channel.orig_h","target_field": "data_channel_source_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "data_channel.resp_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.data_channel.resp_h","target_field": "data_channel_destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "data_channel.resp_p", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.data_channel.resp_p","target_field": "data_channel_destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fuid",		"target_field": "fuid",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_http

@@ -0,0 +1,42 @@
+{
+  "description" : "bro_http",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.trans_depth",	"target_field": "trans_depth",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.method", 		"target_field": "method",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host", 		"target_field": "virtual_host",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uri", 		"target_field": "uri",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.referrer", 	"target_field": "referrer",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version", 		"target_field": "version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user_agent", 	"target_field": "useragent",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_body_len", "target_field": "request_body_length",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response_body_len","target_field": "response_body_length",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_code",	"target_field": "status_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_msg", 	"target_field": "status_message",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.info_code", 	"target_field": "info_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.info_msg", 	"target_field": "info_message",		"ignore_missing": true 	} },
+    { "remove":		{ "field": "message2.tags",		"ignore_failure": true						} },
+    { "rename": 	{ "field": "message2.username", 	"target_field": "user",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.password", 	"target_field": "password",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proxied", 		"target_field": "proxied",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_fuids",	"target_field": "orig_fuids",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_filenames",	"target_field": "orig_filenames",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_mime_types",	"target_field": "orig_mime_types",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_fuids",	"target_field": "resp_fuids",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_filenames",	"target_field": "resp_filenames",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_mime_types",	"target_field": "resp_mime_types",	"ignore_missing": true 	} },
+    { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			"ignore_failure": true 	} },
+    { "script":	{ "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()",	"ignore_failure": true 	} },
+    { "script":	{ "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", 	"ignore_failure": true	} },
+    { "pipeline":       { "name": "bro_common"											} }
+  ]
+}

salt/elasticsearch/files/ingest/bro_intel

@@ -0,0 +1,29 @@
+{
+  "description" : "bro_intel",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "seen.indicator", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.seen.indicator", 	"target_field": "indicator",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "seen.indicator_type", 	"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.seen.indicator_type", 	"target_field": "indicator_type",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "seen.where", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.seen.where", 	"target_field": "seen_where",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "seen.node", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.seen.node", 	"target_field": "seen_node",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.matched", 		"target_field": "matched",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sources", 		"target_field": "sources",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fuid", 		"target_field": "fuid",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.file_mime_type", 	"target_field": "mimetype",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.file_desc", 	"target_field": "file_description",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_irc

@@ -0,0 +1,25 @@
+{
+  "description" : "bro_irc",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.nick", 		"target_field": "nick",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user", 		"target_field": "irc_username",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.command", 		"target_field": "irc_command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.value", 		"target_field": "value",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.addl", 		"target_field": "additional_info",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dcc_file_name", 	"target_field": "dcc_file_name",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dcc_file_size", 	"target_field": "dcc_file_size",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dcc_mime_type", 	"target_field": "dcc_mime_type",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fuid", 		"target_field": "fuid",			"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_kerberos

@@ -0,0 +1,30 @@
+{
+  "description" : "bro_kerberos",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_type", 	"target_field": "request_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client", 		"target_field": "client",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.service", 		"target_field": "service",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.success", 		"target_field": "kerberos_success",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.error_msg", 	"target_field": "error_message",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.from", 		"target_field": "valid_from",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.till", 		"target_field": "valid_till",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cipher", 		"target_field": "cipher",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.forwardable", 	"target_field": "forwardable",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.renewable", 	"target_field": "renewable",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_cert_subject", 	"target_field": "client_certificate_subject",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_cert_fuid", 	"target_field": "client_certificate_fuid",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_cert_subject", 	"target_field": "server_certificate_subject",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_cert_fuid", 	"target_field": "server_certificate_fuid",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_modbus

@@ -0,0 +1,18 @@
+{
+  "description" : "bro_modbus",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.func", 		"target_field": "function",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.exception", 	"target_field": "exception",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_mysql

@@ -0,0 +1,21 @@
+{
+  "description" : "bro_mysql",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cmd", 		"target_field": "mysql_command",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.arg", 		"target_field": "mysql_argument",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.success", 		"target_field": "mysql_success",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rows", 		"target_field": "rows",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response",		"target_field": "response",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_notice

@@ -0,0 +1,36 @@
+{
+  "description" : "bro_notice",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "remove":		{ "field": "message2.dst",		"ignore_failure": true						} },
+    { "remove":		{ "field": "message2.src",		"ignore_failure": true						} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fuid", 		"target_field": "fuid",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mime", 		"target_field": "file_mime_type",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.desc", 		"target_field": "file_description",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.note", 		"target_field": "note",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.msg", 		"target_field": "msg",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sub", 		"target_field": "sub_msg",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.p", 		"target_field": "p",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.n", 		"target_field": "n",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.peer_descr",	"target_field": "peer_description",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.actions", 		"target_field": "action",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.suppress_for", 	"target_field": "suppress_for",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dropped", 		"target_field": "dropped",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_country_code",	"target_field": "destination_country_code",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_region", 	"target_field": "destination_region",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_city", 	"target_field": "destination_city",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_latitude", 	"target_field": "destination_latitude",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_longitude", 	"target_field": "destination_longitude",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_ntlm

@@ -0,0 +1,24 @@
+{
+  "description" : "bro_ntlm",
+  "processors" : [
+    { "json":		{ "field": "message",				"target_field": "message2",			"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 		"target_field": "uid",				"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 			"path": "message2", 				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 		"target_field": "source_ip",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 			"path": "message2",				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 		"target_field": "source_port",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 			"path": "message2",				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 		"target_field": "destination_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 			"path": "message2",				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 		"target_field": "destination_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.hostname",			"target_field": "hostname",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.domainname",		"target_field": "domain_name",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.success", 			"target_field": "ntlm_success",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status", 			"target_field": "status",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.username",			"target_field": "username",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_dns_computer_name",	"target_field": "server_dns_computer_name", 	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_nb_computer_name",	"target_field": "server_nb_computer_name", 	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_tree_name",		"target_field": "server_tree_name",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   		} }
+  ]
+}

salt/elasticsearch/files/ingest/bro_pe

@@ -0,0 +1,23 @@
+{
+  "description" : "bro_pe",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.id", 	 	"target_field": "fuid",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.machine",		"target_field": "machine",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.compile_ts",	"target_field": "compile_ts",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.os", 		"target_field": "os",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.subsystem",	"target_field": "subsystem",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.is_exe", 		"target_field": "is_exe",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.is_64bit",		"target_field": "is_64bit",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uses_aslr", 	"target_field": "uses_aslr",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uses_dep", 	"target_field": "uses_dep",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uses_code_integrity","target_field": "uses_code_integrity","ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uses_seh",		"target_field": "uses_seh",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.has_import_table", "target_field": "has_import_table",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.has_export_table", "target_field": "has_export_table",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.has_cert_table", 	"target_field": "has_cert_table",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.has_debug_data", 	"target_field": "has_debug_data",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.section_names", 	"target_field": "section_names",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_radius

@@ -0,0 +1,25 @@
+{
+  "description" : "bro_radius",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.username", 	"target_field": "username",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mac", 		"target_field": "mac",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.framed_addr", 	"target_field": "framed_addr",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.remote_ip", 	"target_field": "remote_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.connect_info", 	"target_field": "connect_info",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.reply_msg", 	"target_field": "reply_message",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.result", 		"target_field": "result",		"ignore_missing": true 	} },
+    { "remove":		{ "field": "message2.ttl",		"ignore_failure": true						} },
+    { "rename": 	{ "field": "message2.logged", 		"target_field": "logged",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_rdp

@@ -0,0 +1,31 @@
+{
+  "description" : "bro_rdp",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cookie", 		"target_field": "cookie",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.result", 		"target_field": "result",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.security_protocol","target_field": "security_protocol",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.keyboard_layout", 	"target_field": "keyboard_layout",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_build", 	"target_field": "client_build",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_name", 	"target_field": "client_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_dig_product_id", 	"target_field": "client_digital_product_id",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.desktop_width", 	"target_field": "desktop_width",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.desktop_height", 	"target_field": "desktop_height",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.requested_color_depth", 	"target_field": "requested_color_depth",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cert_type", 	"target_field": "certificate_type",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cert_count", 	"target_field": "certificate_count",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cert_permanent", 	"target_field": "certificate_permanent","ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.encryption_level",	"target_field": "encryption_level",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.encryption_method","target_field": "encryption_method",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_rfb

@@ -0,0 +1,26 @@
+{
+  "description" : "bro_rfb",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_major_version",	"target_field": "client_major_version",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_minor_version",	"target_field": "client_minor_version",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_major_version",	"target_field": "server_major_version",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_minor_version",	"target_field": "server_minor_version",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.authentication_method",	"target_field": "authentication_method","ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.auth", 			"target_field": "auth",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.share_flag", 	"target_field": "share_flag",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.desktop_name", 	"target_field": "desktop_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.width", 		"target_field": "width",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.height", 		"target_field": "height",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_signatures

@@ -0,0 +1,22 @@
+{
+  "description" : "bro_signatures",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.note", 		"target_field": "note",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sig_id", 		"target_field": "signature_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.event_msg", 	"target_field": "event_message",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sub_msg", 		"target_field": "sub_message",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sig_count", 	"target_field": "signature_count",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host_count", 	"target_field": "host_count",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_sip

@@ -0,0 +1,37 @@
+{
+  "description" : "bro_sip",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.trans_depth", 	"target_field": "trans_depth",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.method", 		"target_field": "method",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uri", 		"target_field": "uri",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.date", 		"target_field": "date",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_from", 	"target_field": "request_from",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_to", 	"target_field": "request_to",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response_from", 	"target_field": "response_from",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response_to", 	"target_field": "response_to",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.reply_to", 	"target_field": "reply_to",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.call_id", 		"target_field": "call_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.seq", 		"target_field": "seq",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.subject", 		"target_field": "subject",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_path", 	"target_field": "request_path",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response_path", 	"target_field": "response_path",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user_agent", 	"target_field": "user_agent",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_code", 	"target_field": "status_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_msg",	"target_field": "status_msg",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.warning", 		"target_field": "warning",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_body_len",	"target_field": "request_body_length",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response_body_len","target_field": "response_body_length",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.content_type", 	"target_field": "content_type",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_smb_files

@@ -0,0 +1,31 @@
+{
+  "description" : "bro_smb_files",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fuid", 		"target_field": "fuid",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.action", 		"target_field": "action",		"ignore_missing": true 	} },
+    { "remove":		{ "field": "path",			"ignore_failure": true						} },
+    { "rename": 	{ "field": "message2.path", 		"target_field": "path",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.name", 		"target_field": "name",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.size", 		"target_field": "size",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.prev_name", 	"target_field": "prev_name",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "times.modified", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.times.modified", 	"target_field": "times_modified",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "times.accessed", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.times.accessed", 	"target_field": "times_accessed",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "times.created", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.times.created", 	"target_field": "times_created",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "times.changed", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.times.changed", 	"target_field": "times_changed",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_smb_mapping

@@ -0,0 +1,21 @@
+{
+  "description" : "bro_smb_files",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "remove":		{ "field": "path",			"ignore_failure": true						} },
+    { "rename": 	{ "field": "message2.path", 		"target_field": "path",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.service", 		"target_field": "service",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.native_file_system",	"target_field": "native_file_system",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.share_type",	"target_field": "share_type",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_smtp

@@ -0,0 +1,38 @@
+{
+  "description" : "bro_smtp",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "remove":		{ "field": "path",			"ignore_failure": true						} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.trans_depth", 	"target_field": "trans_depth",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.helo", 		"target_field": "helo",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mailfrom", 	"target_field": "mail_from",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rcptto", 		"target_field": "recipient_to",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.date", 		"target_field": "mail_date",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.from", 		"target_field": "from",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.to", 		"target_field": "to",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cc", 		"target_field": "cc",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.reply_to", 	"target_field": "reply_to",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.msg_id", 		"target_field": "message_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.in_reply_to", 	"target_field": "in_reply_to",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.subject", 		"target_field": "subject",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.x_originating_ip",	"target_field": "x_originating_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.first_received", 	"target_field": "first_received",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.second_received",	"target_field": "second_received",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.last_reply",	"target_field": "last_reply",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.path", 		"target_field": "path",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user_agent", 	"target_field": "useragent",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tls", 		"target_field": "tls",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fuids", 		"target_field": "fuids",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.is_webmail",	"target_field": "is_webmail",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_snmp

@@ -0,0 +1,25 @@
+{
+  "description" : "bro_snmp",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.duration",		"target_field": "duration",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version", 		"target_field": "version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.community", 	"target_field": "community",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.get_requests", 	"target_field": "get_requests",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.get_bulk_requests","target_field": "get_bulk_requests",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.get_responses",	"target_field": "get_responses",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.set_requests", 	"target_field": "set_requests",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.display_string", 	"target_field": "display_string",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.up_since", 	"target_field": "up_since",		"ignore_missing": true 	} },
+    { "pipeline":	{ "name": "bro_common"											} }
+  ]
+}

salt/elasticsearch/files/ingest/bro_socks

@@ -0,0 +1,28 @@
+{
+  "description" : "bro_socks",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version", 		"target_field": "version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user", 		"target_field": "user",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.password", 	"target_field": "password",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status", 		"target_field": "status",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_host", 	"target_field": "request_host",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "request.name", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.request.name", 	"target_field": "request_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_p", 	"target_field": "request_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "bound.host", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.bound.host", 	"target_field": "bound_host",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.bound_name", 	"target_field": "bound_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.bound_p", 		"target_field": "bound_port",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_software

@@ -0,0 +1,23 @@
+{
+  "description" : "bro_software",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "version.major", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.version.major", 	"target_field": "version_major",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "version.minor", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.version.minor", 	"target_field": "version_minor",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "version.minor2", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.version.minor2", 	"target_field": "version_minor2",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "version.minor3", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.version.minor3", 	"target_field": "version_minor3",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "version.addl", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.version.addl", 	"target_field": "version_additional_info",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host", 		"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host_p", 		"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.software_type", 	"target_field": "software_type",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.name", 		"target_field": "name",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.unparsed_version",	"target_field": "unparsed_version",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_ssh

@@ -0,0 +1,40 @@
+{
+  "description" : "bro_conn",
+  "processors" : [
+    { "json":		{ "field": "message",				"target_field": "message2",			"ignore_failure": true	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 			"path": "message2", 				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 		"target_field": "source_ip",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 			"path": "message2",				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 		"target_field": "source_port",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 			"path": "message2",				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 		"target_field": "destination_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 			"path": "message2",				"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 		"target_field": "destination_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version", 			"target_field": "version",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uid", 			"target_field": "uid",				"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.hassh", 			"target_field": "hassh",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.auth_success",		"target_field": "authentication_success",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.auth_attempts",		"target_field": "authentication_attempts",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.direction", 		"target_field": "direction",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client", 			"target_field": "client",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server", 			"target_field": "server",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cipher_alg", 		"target_field": "cipher_algorithm",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.compression_alg", 		"target_field": "compression_algorithm",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cshka", 			"target_field": "client_host_key_algorithms",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host_key_alg", 		"target_field": "host_key_algorithm",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.hasshAlgorithms", 		"target_field": "hassh_algorithms",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.hasshServer", 		"target_field": "hassh_server",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.hasshVersion", 		"target_field": "hassh_version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.kex_alg", 			"target_field": "kex_algorithm",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mac_alg", 			"target_field": "mac_algorithm",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sshka", 			"target_field": "server_host_key_algorithms",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host_key", 		"target_field": "host_key",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_region", 	"target_field": "destination_region",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_city", 	"target_field": "destination_city",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_latitude", 	"target_field": "destination_latitude",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_longitude", 	"target_field": "destination_longitude",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.destination_country_code", "target_field": "destination_country_code",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.hasshServerAlgorithms", 	"target_field": "hassh_server_algorithms",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   		} }
+  ]
+}

salt/elasticsearch/files/ingest/bro_ssl

@@ -0,0 +1,33 @@
+{
+  "description" : "bro_ssl",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version", 		"target_field": "version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cipher", 		"target_field": "cipher",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.curve", 		"target_field": "curve",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_name", 	"target_field": "server_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resumed", 		"target_field": "resumed",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.last_alert", 	"target_field": "last_alert",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.next_protocol", 	"target_field": "next_protocol",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.established", 	"target_field": "established",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cert_chain_fuids", 	"target_field": "certificate_chain_fuids",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_cert_chain_fuids",	"target_field": "client_certificate_chain_fuids",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.subject", 		"target_field": "certificate_subject",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.issuer", 		"target_field": "certificate_issuer",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_subject", 	"target_field": "client_subject",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_issuer", 	"target_field": "client_issuer",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.validation_status","target_field": "validation_status",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.ja3", 		"target_field": "ja3",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.ja3s", 		"target_field": "ja3s",			"ignore_missing": true 	} },
+    { "pipeline":       { "name":  "bro_common_ssl"                                                                             } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_syslog

@@ -0,0 +1,21 @@
+{
+  "description" : "bro_syslog",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.facility", 	"target_field": "facility",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.severity", 	"target_field": "severity",		"ignore_missing": true 	} },
+    { "remove":		{ "field": "message",			"ignore_failure": true						} },
+    { "rename": 	{ "field": "message2.message", 		"target_field": "message",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_tunnel

@@ -0,0 +1,7 @@
+{
+  "description" : "bro_tunnel",
+  "processors" : [
+    { "set":		{ "field": "event_type", "value": "bro_tunnels"								} },
+    { "pipeline":       { "name": "bro_tunnels"                                                                                 } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_tunnels

@@ -0,0 +1,18 @@
+{
+  "description" : "bro_tunnels",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tunnel_type", 	"target_field": "tunnel_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.action", 		"target_field": "action",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_weird

@@ -0,0 +1,20 @@
+{
+  "description" : "bro_weird",
+  "processors" : [
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.name", 		"target_field": "name",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.addl", 		"target_field": "additional_info",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.notice", 		"target_field": "notice",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.peer", 		"target_field": "peer",			"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/bro_x509

@@ -0,0 +1,44 @@
+{
+  "description" : "bro_x509",
+  "processors" : [
+    { "json":		{ "field": "message",					"target_field": "message2",				"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.id", 	 			"target_field": "id",					"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.version",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.version", 		"target_field": "certificate_version",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.serial",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.serial", 		"target_field": "certificate_serial",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.subject",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.subject", 		"target_field": "certificate_subject",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.issuer",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.issuer", 		"target_field": "certificate_issuer",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.not_valid_before",		"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.not_valid_before", 	"target_field": "certificate_not_valid_before",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.not_valid_after",		"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.not_valid_after", 	"target_field": "certificate_not_valid_after",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.key_alg",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.key_alg", 		"target_field": "certificate_key_algorithm",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.sig_alg",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.sig_alg", 		"target_field": "certificate_signing_algorithm",	"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.key_type",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.key_type", 		"target_field": "certificate_key_type",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.key_length",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.key_length", 		"target_field": "certificate_key_length",		"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.exponent",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.exponent", 		"target_field": "certificate_exponent",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "certificate.curve",				"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.certificate.curve", 		"target_field": "certificate_curve",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "san.dns",					"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.san.dns", 				"target_field": "san_dns",				"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "san.uri",					"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.san.uri", 				"target_field": "san_uri",				"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "san.email",					"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.san.email", 			"target_field": "san_email",				"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "san.ip",					"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.san.ip", 				"target_field": "san_ip",				"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "basic_constraints.ca",			"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.basic_constraints.ca", 		"target_field": "basic_constraints_ca",			"ignore_missing": true 	} },
+    { "dot_expander": 	{ "field": "basic_constraints.path_length",		"path": "message2", 					"ignore_failure": true 	} },
+    { "rename": 	{ "field": "message2.basic_constraints.path_length", 	"target_field": "basic_constraints_path_length",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "bro_common_ssl"                                                                                   				} }
+  ]
+}

salt/elasticsearch/files/ingest/common

@@ -0,0 +1,52 @@
+{
+  "description" : "common",
+  "processors" : [
+    { 
+	"rename": {
+		"field": "type",
+		"target_field": "event_type",
+		"ignore_missing": true
+	}
+    },
+    { 	
+	"geoip":  { 
+		"field": "destination_ip",
+		"target_field": "destination_geo",
+		"database_file": "GeoLite2-City.mmdb",
+		"ignore_missing": true, 
+		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
+	}
+    },
+    { 
+	"geoip":  {
+		"field": "source_ip",
+		"target_field": "source_geo",
+		"database_file": "GeoLite2-City.mmdb",
+		"ignore_missing": true,
+		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
+	}
+    },
+    { 
+	"split":  {
+		"field": "_index",
+		"target_field": "index_name_prefix",
+		"separator": "-"
+	}
+    },
+    { 
+	"date_index_name": {
+        	"field": "@timestamp",
+	        "index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-",
+        	"date_rounding": "d",
+	        "ignore_failure": true,
+        	"index_name_format": "yyyy.MM.dd"
+        }
+    },
+    { 
+	"remove": {
+		"field": "index_name_prefix",
+		"ignore_failure": true
+	}
+    }
+  ]
+}

salt/elasticsearch/files/ingest/common_nids

@@ -0,0 +1,17 @@
+{
+  "description" : "common_nids",
+  "processors" : [
+    { "convert": 	{ "field": "sid", 			"type": "integer" 										} },
+    { "set": 		{ "if": "ctx.sid < 1000000",		"field": "signature_info", "value": "https://www.snort.org/search?query={{gid}}-{{sid}}"	} },
+    { "set": 		{ "if": "ctx.sid > 1999999",		"field": "signature_info", "value": "https://doc.emergingthreats.net/{{sid}}"			} },
+    { "remove":		{ "if": "ctx.sid > 2999999",		"field": "signature_info" 									} },
+    { "set": 		{ "if": "ctx.priority == '1'",		"field": "severity", "value": "High"	 							} },
+    { "set": 		{ "if": "ctx.priority == '2'",		"field": "severity", "value": "Medium" 								} },
+    { "set": 		{ "if": "ctx.priority == '3'",		"field": "severity", "value": "Low" 								} },
+    { "dissect": 	{ "field": "alert",			"pattern" : "%{rule_type} %{category} ",				"ignore_failure": true 	} },
+    { "set": 		{ "if": "ctx.rule_type == 'GPL'",	"field": "rule_type", "value": "Snort GPL"							} },
+    { "set": 		{ "if": "ctx.rule_type == 'ET'",	"field": "rule_type", "value": "Emerging Threats"						} },
+    { "lowercase": 	{ "field": "category",												"ignore_failure": true	} },
+    { "pipeline":       { "name": "common"															} }
+  ]
+}

salt/elasticsearch/files/ingest/ossec

@@ -0,0 +1,53 @@
+{
+  "description" : "ossec",
+  "processors" : [
+    { "json":		{ "field": "message",					"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.agent", 				"target_field": "agent",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data", 				"target_field": "data",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.decoder", 				"target_field": "decoder",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.full_log", 			"target_field": "full_log",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id", 				"target_field": "id",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.location", 			"target_field": "location",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.manager", 				"target_field": "manager",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.predecoder", 			"target_field": "predecoder",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.timestamp", 			"target_field": "timestamp",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rule", 				"target_field": "wazuh-rule",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.command", 				"target_field": "command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.dstip", 				"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.dstport", 				"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.dstuser", 				"target_field": "escalated_user",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.srcip", 				"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.srcuser", 				"target_field": "username",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.destinationHostname", 	"target_field": "destination_hostname",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.destinationIp", 		"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.destinationPort", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.image", 			"target_field": "image_path",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.parentImage", 		"target_field": "parent_image_path",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.sourceHostname", 	"target_field": "source_hostname",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.sourceIp", 		"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.sourcePort", 		"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.targetFilename",		"target_field": "target_filename",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.user", 			"target_field": "username",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.system.eventID", 			"target_field": "event_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "predecoder.program_name", 			"target_field": "process",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "wazuh-rule.level", 				"target_field": "alert_level",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "wazuh-rule.description", 			"target_field": "description",		"ignore_missing": true 	} },
+    { "set": 		{ "if": "ctx.alert_level == 1", 		"field": "classification", "value": "None" 				} },
+    { "set": 		{ "if": "ctx.alert_level == 2", 		"field": "classification", "value": "System low priority notification" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 3", 		"field": "classification", "value": "Successful/authorized event" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 4", 		"field": "classification", "value": "System low priority error" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 5", 		"field": "classification", "value": "User generated error" 		} },
+    { "set": 		{ "if": "ctx.alert_level == 6", 		"field": "classification", "value": "Low relevance attack" 		} },
+    { "set": 		{ "if": "ctx.alert_level == 7", 		"field": "classification", "value": "\"Bad word\" matching" 		} },
+    { "set": 		{ "if": "ctx.alert_level == 8", 		"field": "classification", "value": "First time seen" 			} },
+    { "set": 		{ "if": "ctx.alert_level == 9", 		"field": "classification", "value": "Error from invalid source" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 10", 		"field": "classification", "value": "Multiple user generated errors" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 11", 		"field": "classification", "value": "Integrity checking warning" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 12", 		"field": "classification", "value": "High importance event" 		} },
+    { "set": 		{ "if": "ctx.alert_level == 13", 		"field": "classification", "value": "Unusal error (high importance)" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 14", 		"field": "classification", "value": "High importance security event" 	} },
+    { "set": 		{ "if": "ctx.alert_level == 15", 		"field": "classification", "value": "Severe attack" 			} },
+    { "append": 	{ "if": "ctx.alert_level != null",		"field": "tags", "value": ["alert"] 					} },
+    { "pipeline":	{ "name": "common" } }
+  ]
+}

salt/elasticsearch/files/ingest/sguild_nids

@@ -0,0 +1,25 @@
+{
+  "description" : "sguild_nids",
+  "processors" : [
+    { 
+	"dissect": {
+		"field": "message",
+		"pattern" : "%{} %{} %{}  Alert Received: %{} %{priority} %{classification} %{interface} {%{alerttime}} %{} %{} {%{alert}} %{source_ip} %{destination_ip} %{protocol} %{source_port} %{destination_port} %{gid} %{sid} %{rev} ",
+		"on_failure": [ { "drop" : { } } ]
+	}
+    },
+    { "set":		{ "if": "ctx.protocol == '1'",		"field": "protocol", "value": "ICMP"							} },
+    { "set":		{ "if": "ctx.protocol == '6'",		"field": "protocol", "value": "TCP" 							} },
+    { "set":		{ "if": "ctx.protocol == '17'",		"field": "protocol", "value": "UDP" 							} },
+    { "remove":		{ "if": "ctx.source_ip == '{}'",  	"field": "source_ip"									} },
+    { "remove":		{ "if": "ctx.destination_ip == '{}'", 	"field": "destination_ip"								} },
+    { "remove":		{ "if": "ctx.protocol == '{}'", 	"field": "protocol"									} },
+    { "remove":		{ "if": "ctx.source_port == '{}'",  	"field": "source_port"									} },
+    { "remove":		{ "if": "ctx.destination_port == '{}'", "field": "destination_port"								} },
+    { "set":		{ "field": "type", "value": "snort" 												} },
+    { "rename":		{ "field": "@timestamp",	"target_field": "timestamp",						"ignore_missing": true  } },
+    { "date":		{ "field": "alerttime",		"target_field": "@timestamp",	"formats": ["yyyy-MM-dd HH:mm:ss"],	"ignore_failure": true  } },
+    { "remove":		{ "field": "alerttime",											"ignore_missing": true  } },
+    { "pipeline": 	{ "name": "common_nids" 													} }
+  ]
+}

salt/elasticsearch/files/ingest/snort

@@ -0,0 +1,21 @@
+{
+  "description" : "snort",
+  "processors" : [
+    { 
+	"dissect": {
+		"field": "message",
+		"pattern" : "[%{gid}:%{sid}:%{rev}] %{alert} [Classification: %{classification}] [Priority: %{priority}]: <%{interface}> {%{protocol}} %{source_ip_port} -> %{destination_ip_port}",
+		"on_failure": [ { "drop" : { } } ]
+	}
+    },
+    { "split": { "field": "source_ip_port", 		"separator": ":", 			"ignore_failure": true 	} },
+    { "split": { "field": "destination_ip_port", 	"separator": ":", 			"ignore_failure": true 	} },
+    { "rename":{ "field": "source_ip_port.1",		"target_field": "source_port",		"ignore_failure": true 	} },
+    { "rename":{ "field": "destination_ip_port.1",	"target_field": "destination_port",	"ignore_failure": true 	} },
+    { "rename":{ "field": "source_ip_port.0",		"target_field": "source_ip",		"ignore_failure": true 	} },
+    { "rename":{ "field": "destination_ip_port.0",	"target_field": "destination_ip",	"ignore_failure": true 	} },
+    { "remove":{ "field": "source_ip_port", 							"ignore_failure": true	} },
+    { "remove":{ "field": "destination_ip_port", 						"ignore_failure": true	} },
+    { "pipeline": { "name": "common_nids" } }
+  ]
+}

salt/elasticsearch/files/so-elasticsearch-pipelines

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+ELASTICSEARCH_HOST=$1
+ELASTICSEARCH_PORT=9200
+
+# Define a default directory to load pipelines from
+ELASTICSEARCH_INGEST_PIPELINES="/opt/so/saltstack/salt/elasticsearch/files/ingest/"
+
+# Wait for ElasticSearch to initialize
+echo -n "Waiting for ElasticSearch..."
+COUNT=0
+ELASTICSEARCH_CONNECTED="no"
+while [[ "$COUNT" -le 240 ]]; do
+	curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+	if [ $? -eq 0 ]; then
+		ELASTICSEARCH_CONNECTED="yes"
+		echo "connected!"
+		break
+	else
+		((COUNT+=1))
+		sleep 1
+		echo -n "."
+	fi
+done
+if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+	echo
+	echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'" 
+	echo
+fi
+
+cd ${ELASTICSEARCH_INGEST_PIPELINES}
+
+echo "Loading pipelines..."
+for i in *; do echo $i; curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
+echo
+
+cd - >/dev/null

salt/elasticsearch/init.sls

@@ -121,6 +121,10 @@ so-elasticsearch:
       - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
       - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
 
+so-elasticsearch-pipelines:
+ cmd.run:
+   - name: /opt/so/saltstack/salt/elasticsearch/files/so-elasticsearch-pipelines {{ esclustername }}
+
 # Tell the main cluster I am here
 #curl -XPUT http://\$ELASTICSEARCH_HOST:\$ELASTICSEARCH_PORT/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"$HOSTNAME": {"skip_unavailable": "true", "seeds": ["$DOCKER_INTERFACE:$REVERSE_PORT"]}}}}}'
 

salt/filebeat/etc/filebeat.yml

@@ -6,6 +6,59 @@
 
 name: {{ HOSTNAME }}
 
+#================================ Logging ======================================
+# There are four options for the log output: file, stderr, syslog, eventlog
+# The file output is the default.
+
+# Sets log level. The default log level is info.
+# Available log levels are: error, warning, info, debug
+logging.level: debug
+
+# Enable debug output for selected components. To enable all selectors use ["*"]
+# Other available selectors are "beat", "publish", "service"
+# Multiple selectors can be chained.
+#logging.selectors: [ ]
+
+# Send all logging output to syslog. The default is false.
+#logging.to_syslog: false
+
+# Send all logging output to Windows Event Logs. The default is false.
+#logging.to_eventlog: false
+
+# If enabled, filebeat periodically logs its internal metrics that have changed
+# in the last period. For each metric that changed, the delta from the value at
+# the beginning of the period is logged. Also, the total values for
+# all non-zero internal metrics are logged on shutdown. The default is true.
+#logging.metrics.enabled: true
+
+# The period after which to log the internal metrics. The default is 30s.
+#logging.metrics.period: 30s
+
+# Logging to rotating files. Set logging.to_files to false to disable logging to
+# files.
+logging.to_files: true
+logging.files:
+  # Configure the path where the logs are written. The default is the logs directory
+  # under the home path (the binary location).
+  path: /usr/share/filebeat/logs
+
+  # The name of the files where the logs are written to.
+  name: filebeat.log
+
+  # Configure log file size limit. If limit is reached, log file will be
+  # automatically rotated
+  rotateeverybytes: 10485760 # = 10MB
+
+  # Number of rotated log files to keep. Oldest files will be deleted first.
+  keepfiles: 7
+
+  # The permissions mask to apply when rotating log files. The default value is 0600.
+  # Must be a valid Unix-style file permissions mask expressed in octal notation.
+  #permissions: 0600
+
+# Set to true to log messages in json format.
+#logging.json: false
+
 #==========================  Modules configuration ============================
 filebeat.modules:
 #=========================== Filebeat prospectors =============================
@@ -181,58 +234,6 @@ output.logstash:
 
 
 
-#================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-logging.level: debug
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, filebeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/filebeat
-
-  # The name of the files where the logs are written to.
-  name: filebeat.log
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 7
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
 
 
 #============================== Xpack Monitoring =====================================

salt/filebeat/init.sls

@@ -23,6 +23,13 @@ filebeatetcdir:
     - group: 939
     - makedirs: True
 
+filebeatlogdir:
+  file.directory:
+    - name: /opt/so/log/filebeat
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 filebeatpkidir:
   file.directory:
     - name: /opt/so/conf/filebeat/etc/pki
@@ -51,18 +58,18 @@ filebeatconfsync:
 
 so-filebeatimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-filebeat:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-filebeat:HH1.1.1
 
 so-filebeat:
   docker_container.running:
     - require:
       - so-filebeatimage
-    - image: soshybridhunter/so-filebeat:HH1.1.0
+    - image: soshybridhunter/so-filebeat:HH1.1.1
     - hostname: so-filebeat
     - user: root
     - extra_hosts: {{ MASTER }}:{{ MASTERIP }}
     - binds:
-      - /opt/so/log/filebeat:/var/log/filebeat:rw
+      - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
       - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
       - /nsm/bro:/nsm/bro:ro
       - /opt/so/log/suricata:/suricata:ro

salt/firewall/init.sls

@@ -228,6 +228,28 @@ enable_master_osquery_8080_{{ip}}:
     - position: 1
     - save: True
 
+enable_master_playbook_3200_{{ip}}:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - proto: tcp
+    - source: {{ ip }}
+    - dport: 3200
+    - position: 1
+    - save: True
+    
+enable_master_navigator_4200_{{ip}}:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - proto: tcp
+    - source: {{ ip }}
+    - dport: 4200
+    - position: 1
+    - save: True
+
 {% endfor %}
 
 # Make it so all the minions can talk to salt and update etc.

salt/hive/init.sls

@@ -33,13 +33,13 @@ hiveesdata:
 
 so-thehive-esimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive-es:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive-es:HH1.1.1
 
 so-thehive-es:
   docker_container.running:
     - require:
       - so-thehive-esimage
-    - image: soshybridhunter/so-thehive-es:HH1.1.0
+    - image: soshybridhunter/so-thehive-es:HH1.1.1
     - hostname: so-thehive-es
     - name: so-thehive-es
     - user: 939
@@ -56,7 +56,6 @@ so-thehive-es:
       - transport.tcp.port=9500
       - transport.host=0.0.0.0
       - cluster.name=hive
-      - script.inline=true
       - thread_pool.index.queue_size=100000
       - thread_pool.search.queue_size=100000
       - thread_pool.bulk.queue_size=100000
@@ -81,13 +80,13 @@ so-thehive-es:
 
 so-thehiveimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-thehive:HH1.1.1
 
 so-thehive:
   docker_container.running:
     - require:
       - so-thehiveimage
-    - image: soshybridhunter/so-thehive:HH1.1.0
+    - image: soshybridhunter/so-thehive:HH1.1.1
     - environment:
       - ELASTICSEARCH_HOST={{ MASTERIP }}
     - hostname: so-thehive
@@ -97,8 +96,9 @@ so-thehive:
       - /opt/so/conf/hive/etc/application.conf:/opt/thehive/conf/application.conf:ro
     - port_bindings:
       - 0.0.0.0:9000:9000
-    
+
 hivescript:
   cmd.script:
     - source: salt://hive/thehive/scripts/hive_init.sh
+    - cwd: /opt/so
     - template: jinja

salt/hive/thehive/etc/application.conf

@@ -5,59 +5,57 @@
 # WARNING: If you deploy your application on several servers, make sure to use the same key.
 play.http.secret.key="letsdewdis"
 play.http.context=/thehive/
-
+search.uri = "http://{{ MASTERIP }}:9400"
 # Elasticsearch
 search {
-  ## Basic configuration
-  # Index name.
-  index = the_hive
-  # ElasticSearch cluster name.
-  cluster = hive
-  # ElasticSearch instance address.
-  host = ["{{ MASTERIP }}:9500"]
+ # Name of the index
+ index = the_hive
+ # Name of the Elasticsearch cluster
+ cluster = hive
+ # Address of the Elasticsearch instance
+ host = ["{{ MASTERIP }}:9500"]
+ #search.uri = "http://{{ MASTERIP }}:9500"
+ # Scroll keepalive
+ keepalive = 1m
+ # Size of the page for scroll
+ pagesize = 50
+ # Number of shards
+ nbshards = 5
+ # Number of replicas
+ nbreplicas = 1
+ # Arbitrary settings
+ settings {
+   # Maximum number of nested fields
+   mapping.nested_fields.limit = 100
+ }
 
-  ## Advanced configuration
-  # Scroll keepalive.
-  #keepalive = 1m
-  # Scroll page size.
-  #pagesize = 50
-  # Number of shards
-  #nbshards = 5
-  # Number of replicas
-  #nbreplicas = 1
-  # Arbitrary settings
-  #settings {
-  #  # Maximum number of nested fields
-  #  mapping.nested_fields.limit = 100
-  #}
+ ### XPack SSL configuration
+ # Username for XPack authentication
+ #username
+ # Password for XPack authentication
+ #password
+ # Enable SSL to connect to ElasticSearch
+ ssl.enabled = false
+ # Path to certificate authority file
+ #ssl.ca
+ # Path to certificate file
+ #ssl.certificate
+ # Path to key file
+ #ssl.key
 
-  ### XPack SSL configuration
-  # Username for XPack authentication
-  #search.username = ""
-  # Password for XPack authentication
-  #search.password = ""
-  # Enable SSL to connect to ElasticSearch
-  search.ssl.enabled = false
-  # Path to certificate authority file
-  #search.ssl.ca = ""
-  # Path to certificate file
-  #search.ssl.certificate = ""
-  # Path to key file
-  #search.ssl.key = ""
-
-  ### SearchGuard configuration
-  # Path to JKS file containing client certificate
-  #search.guard.keyStore.path = ""
-  # Password of the keystore
-  #search.guard.keyStore.password = ""
-  # Path to JKS file containing certificate authorities
-  #search.guard.trustStore.path = ""
-  ## Password of the truststore
-  #search.guard.trustStore.password = ""
-  # Enforce hostname verification
-  #search.guard.hostVerification = false
-  # If hostname verification is enabled specify if hostname should be resolved
-  #search.guard.hostVerificationResolveHostname = false
+ ### SearchGuard configuration
+ # Path to JKS file containing client certificate
+ #guard.keyStore.path
+ # Password of the keystore
+ #guard.keyStore.password
+ # Path to JKS file containing certificate authorities
+ #guard.trustStore.path
+ ## Password of the truststore
+ #guard.trustStore.password
+ # Enforce hostname verification
+ #guard.hostVerification
+ # If hostname verification is enabled specify if hostname should be resolved
+ #guard.hostVerificationResolveHostname
 }
 
 # Authentication

salt/hive/thehive/etc/es/elasticsearch.yml

@@ -11,7 +11,6 @@ http.host: 0.0.0.0
 http.port: 9400
 transport.tcp.port: 9500
 transport.host: 0.0.0.0
-script.inline: true
 thread_pool.index.queue_size: 100000
 thread_pool.search.queue_size: 100000
 thread_pool.bulk.queue_size: 100000

salt/kibana/init.sls

@@ -56,14 +56,14 @@ synckibanacustom:
 
 so-kibanaimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-kibana:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-kibana:HH1.1.1
 
 # Start the kibana docker
 so-kibana:
   docker_container.running:
     - require:
       - so-kibanaimage
-    - image: soshybridhunter/so-kibana:HH1.1.0
+    - image: soshybridhunter/so-kibana:HH1.1.1
     - hostname: kibana
     - user: kibana
     - environment:

salt/logstash/conf/conf.enabled.txt.parser

@@ -7,11 +7,12 @@
 # /usr/share/logstash/pipeline.custom/1234_input_custom.conf
 ##
 # All of the defaults are loaded.
+# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest.
 /usr/share/logstash/pipeline.dynamic/0900_input_redis.conf
 /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf
 /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf
 /usr/share/logstash/pipeline.so/1002_preprocess_json.conf
-/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf
+#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf
 /usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf
 /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf
 /usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf
@@ -20,44 +21,44 @@
 /usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf
 /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf
 /usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf
-/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
-/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
-/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
-/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
-/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
-/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
-/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
-/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
-/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
-/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
-/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
-/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
-/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
-/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
-/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
-/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
-/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
-/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
-/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
-/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
-/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
-/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
-/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
-/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
-/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
-/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
-/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
-/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
-/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
-/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
-/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
-/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
-/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
-/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
+#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
+#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
+#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
+#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
+#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
+#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
+#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
+#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
+#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
+#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
+#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
+#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
+#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
+#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
+#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
+#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
+#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
+#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
+#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
+#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
+#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
+#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
+#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
+#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
+#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
+#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
+#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
+#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
+#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
+#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
+#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
+#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
+#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
+#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
 /usr/share/logstash/pipeline.so/1998_test_data.conf
 /usr/share/logstash/pipeline.so/2000_network_flow.conf
-/usr/share/logstash/pipeline.so/6000_bro.conf
-/usr/share/logstash/pipeline.so/6001_bro_import.conf
+#/usr/share/logstash/pipeline.so/6000_bro.conf
+#/usr/share/logstash/pipeline.so/6001_bro_import.conf
 /usr/share/logstash/pipeline.so/6002_syslog.conf
 /usr/share/logstash/pipeline.so/6101_switch_brocade.conf
 /usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf
@@ -68,17 +69,17 @@
 /usr/share/logstash/pipeline.so/6500_ossec.conf
 /usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf
 /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf
-/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf
+#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf
 /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf
 /usr/share/logstash/pipeline.so/8006_postprocess_dns.conf
 /usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf
 /usr/share/logstash/pipeline.so/8007_postprocess_http.conf
 /usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf
 /usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf
-/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf
-/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf
-/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf
-/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf
+#/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf
+#/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf
+#/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf
+#/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf
 /usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf
 /usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf
 /usr/share/logstash/pipeline.dynamic/9999_output_redis.conf

salt/logstash/conf/conf.enabled.txt.so-eval

@@ -7,6 +7,7 @@
 # /usr/share/logstash/pipeline.custom/1234_input_custom.conf
 ##
 # All of the defaults are loaded.
+# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest.
 #/usr/share/logstash/pipeline.so/0000_input_syslogng.conf
 #/usr/share/logstash/pipeline.so/0001_input_json.conf
 #/usr/share/logstash/pipeline.so/0002_input_windows_json.conf
@@ -18,7 +19,7 @@
 #/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf
 #/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf
 #/usr/share/logstash/pipeline.so/1002_preprocess_json.conf
-/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf
+#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf
 #/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf
 /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf
 #/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf
@@ -27,44 +28,44 @@
 #/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf
 /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf
 #/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf
-/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
-/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
-/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
-/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
-/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
-/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
-/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
-/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
-/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
-/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
-/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
-/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
-/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
-/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
-/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
-/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
-/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
-/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
-/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
-/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
-/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
-/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
-/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
-/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
-/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
-/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
-/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
-/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
-/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
-/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
-/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
-/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
-/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
-/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
+#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
+#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
+#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
+#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
+#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
+#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
+#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
+#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
+#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
+#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
+#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
+#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
+#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
+#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
+#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
+#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
+#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
+#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
+#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
+#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
+#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
+#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
+#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
+#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
+#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
+#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
+#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
+#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
+#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
+#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
+#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
+#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
+#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
+#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
 #/usr/share/logstash/pipeline.so/1998_test_data.conf
 #/usr/share/logstash/pipeline.so/2000_network_flow.conf
-/usr/share/logstash/pipeline.so/6000_bro.conf
-/usr/share/logstash/pipeline.so/6001_bro_import.conf
+#/usr/share/logstash/pipeline.so/6000_bro.conf
+#/usr/share/logstash/pipeline.so/6001_bro_import.conf
 #/usr/share/logstash/pipeline.so/6002_syslog.conf
 #/usr/share/logstash/pipeline.so/6101_switch_brocade.conf
 #/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf
@@ -77,7 +78,7 @@
 /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf
 /usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf
 /usr/share/logstash/pipeline.so/6700_winlogbeat.conf
-/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf
+#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf
 /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf
 #/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf
 #/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf

salt/logstash/conf/conf.enabled.txt.storage

@@ -7,11 +7,12 @@
 # /usr/share/logstash/pipeline.custom/1234_input_custom.conf
 ##
 # All of the defaults are loaded.
+# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest.
 /usr/share/logstash/pipeline.dynamic/0900_input_redis.conf
 /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf
 /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf
 /usr/share/logstash/pipeline.so/1002_preprocess_json.conf
-/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf
+#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf
 /usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf
 /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf
 /usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf
@@ -20,44 +21,44 @@
 /usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf
 /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf
 /usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf
-/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
-/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
-/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
-/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
-/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
-/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
-/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
-/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
-/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
-/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
-/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
-/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
-/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
-/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
-/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
-/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
-/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
-/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
-/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
-/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
-/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
-/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
-/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
-/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
-/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
-/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
-/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
-/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
-/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
-/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
-/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
-/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
-/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
-/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
+#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
+#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
+#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
+#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
+#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
+#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
+#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
+#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
+#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
+#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
+#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
+#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
+#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
+#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
+#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
+#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
+#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
+#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
+#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
+#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
+#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
+#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
+#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
+#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
+#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
+#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
+#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
+#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
+#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
+#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
+#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
+#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
+#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
+#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
 /usr/share/logstash/pipeline.so/1998_test_data.conf
 /usr/share/logstash/pipeline.so/2000_network_flow.conf
-/usr/share/logstash/pipeline.so/6000_bro.conf
-/usr/share/logstash/pipeline.so/6001_bro_import.conf
+#/usr/share/logstash/pipeline.so/6000_bro.conf
+#/usr/share/logstash/pipeline.so/6001_bro_import.conf
 /usr/share/logstash/pipeline.so/6002_syslog.conf
 /usr/share/logstash/pipeline.so/6101_switch_brocade.conf
 /usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf
@@ -70,7 +71,7 @@
 /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf
 /usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf
 /usr/share/logstash/pipeline.so/6700_winlogbeat.conf
-/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf
+#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf
 /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf
 #/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf
 #/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf

salt/logstash/etc/logstash-template.json

@@ -2675,6 +2675,14 @@
         "rule_number":{
           "type":"long"
         },
+	"rule_signature":{
+          "type":"text",
+          "fields":{
+            "keyword":{
+              "type":"keyword"
+            }
+          }
+        },
         "rule_type":{
           "type":"text",
           "fields":{

salt/logstash/files/dynamic/9000_output_bro.conf

@@ -20,6 +20,7 @@ output {
   if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] {
 #    stdout { codec => rubydebug }
     elasticsearch {
+      pipeline => "%{event_type}"
       hosts => "{{ ES }}"
       index => "logstash-bro-%{+YYYY.MM.dd}"
       template_name => "logstash"

salt/logstash/init.sls

@@ -148,13 +148,13 @@ lslogdir:
 # Add the container
 so-logstashimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-logstash:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-logstash:HH1.1.1
 
 so-logstash:
   docker_container.running:
     - require:
       - so-logstashimage
-    - image: soshybridhunter/so-logstash:HH1.1.0
+    - image: soshybridhunter/so-logstash:HH1.1.1
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash
@@ -193,3 +193,7 @@ so-logstash:
       {%- endif %}
     - watch:
       - file: /opt/so/conf/logstash/etc
+      - file: /opt/so/conf/logstash/conf.enabled.txt
+      - file: /opt/so/conf/logstash/custom
+      #- file: /opt/so/conf/logstash/rulesets
+      - file: /opt/so/conf/logstash/dynamic

salt/pcap/init.sls

@@ -96,13 +96,13 @@ stenolog:
 
 so-stenoimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-steno:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-steno:HH1.1.1
 
 so-steno:
   docker_container.running:
     - require:
       - so-stenoimage
-    - image: soshybridhunter/so-steno:HH1.1.0
+    - image: soshybridhunter/so-steno:HH1.1.1
     - network_mode: host
     - privileged: True
     - port_bindings:
@@ -116,6 +116,6 @@ so-steno:
       - /nsm/pcapout:/nsm/pcapout:rw
       - /opt/so/log/stenographer:/var/log/stenographer:rw
       - /opt/so/conf/steno/sensoroni.json:/opt/sensoroni/sensoroni.json:ro
-      - /opt/so/log/stenographer:/opt/sensoroni/log:rw
+      - /opt/so/log/stenographer:/opt/sensoroni/logs:rw
     - watch:
       - /opt/so/conf/steno/sensoroni.json

salt/playbook/files/nav_layer_playbook.json

@@ -0,0 +1,28 @@
+{
+	"name": "Playbook",
+	"version": "2.1",
+	"domain": "mitre-enterprise",
+	"description": "Current Coverage of Playbook",
+	"filters": {
+		"stages": ["act"],
+		"platforms": [
+			"windows",
+			"linux",
+			"mac"
+		]
+	},
+	"sorting": 0,
+	"viewMode": 0,
+	"hideDisabled": "false",
+	"techniques": [{
+	}],
+	"gradient": {
+		"colors": ["#ff6666", "#ffe766", "#8ec843"],
+		"minValue": 0,
+		"maxValue": 100
+	},
+	"metadata": [],
+	"showTacticRowBackground": "false",
+	"tacticRowBackground": "#dddddd",
+	"selectTechniquesAcrossTactics": "true"
+}

salt/playbook/files/navigator_config.json

@@ -0,0 +1,59 @@
+{%- set ip = salt['pillar.get']('static:masterip', '') %}
+
+{
+    "enterprise_attack_url": "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json",
+    "pre_attack_url": "https://raw.githubusercontent.com/mitre/cti/master/pre-attack/pre-attack.json",
+    "mobile_data_url": "https://raw.githubusercontent.com/mitre/cti/master/mobile-attack/mobile-attack.json",
+    "taxii_server": {
+        "enabled": false,
+        "url": "https://cti-taxii.mitre.org/",
+        "collections": {
+            "enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e",
+            "pre_attack": "062767bd-02d2-4b72-84ba-56caef0f8658",
+            "mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b"
+        }
+    },
+
+    "domain": "mitre-enterprise",
+
+  "custom_context_menu_items": [ {"label": "view related plays","url": "  https://{{ip}}/playbook/projects/playbook-prod/issues?utf8=%E2%9C%93&set_filter=1&sort=id%3Adesc&f%5B%5D=status_id&op%5Bstatus_id%5D=o&f%5B%5D=cf_27&op%5Bcf_27%5D=%3D&f%5B%5D=&c%5B%5D=status&c%5B%5D=cf_24&c%5B%5D=cf_25&c%5B%5D=cf_6&c%5B%5D=updated_on&group_by=&t%5B%5D=&v%5Bcf_27%5D%5B%5D=~Technique_ID~"}],
+
+"default_layers": {
+     "enabled": true,
+     "urls": [
+         "assets/playbook.json" 
+            ]
+ },
+
+    "comment_color": "yellow",
+
+    "features": [
+        {"name": "tabs", "enabled": true, "description": "Disable to remove the ability to open new tabs."},
+        {"name": "selecting_techniques", "enabled": true, "description": "Disable to remove the ability to select techniques."},
+        {"name": "header", "enabled": true, "description": "Disable to remove the header containing 'MITRE ATT&CK Navigator' and the link to the help page. The help page can still be accessed from the new tab menu."},
+        {"name": "selection_controls", "enabled": true, "description": "Disable to to disable all subfeatures", "subfeatures": [
+            {"name": "search", "enabled": true, "description": "Disable to remove the technique search panel from the interface."},
+            {"name": "multiselect", "enabled": true, "description": "Disable to remove the multiselect panel from interface."},
+            {"name": "deselect_all", "enabled": true, "description": "Disable to remove the deselect all button from the interface."}
+        ]},
+        {"name": "layer_controls", "enabled": true, "description": "Disable to to disable all subfeatures", "subfeatures": [
+            {"name": "layer_info", "enabled": true, "description": "Disable to remove the layer info (name, description and metadata) panel from the interface. Note that the layer can still be renamed in the tab."},
+            {"name": "download_layer", "enabled": true, "description": "Disable to remove the button to download the layer."},
+            {"name": "export_render", "enabled": true, "description": "Disable to the remove the button to render the current layer."},
+            {"name": "export_excel", "enabled": true, "description": "Disable to the remove the button to export the current layer to MS Excel (.xlsx) format."},
+            {"name": "filters", "enabled": true, "description": "Disable to the remove the filters panel from interface."},
+            {"name": "sorting", "enabled": true, "description": "Disable to the remove the sorting button from the interface."},
+            {"name": "color_setup", "enabled": true, "description": "Disable to the remove the color setup panel from interface, containing customization controls for scoring gradient and tactic row color."},
+            {"name": "toggle_hide_disabled", "enabled": true, "description": "Disable to the remove the hide disabled techniques button from the interface."},
+            {"name": "toggle_view_mode", "enabled": true, "description": "Disable to the remove the toggle view mode button from interface."},
+            {"name": "legend", "enabled": true, "description": "Disable to the remove the legend panel from the interface."}
+        ]},
+        {"name": "technique_controls", "enabled": true, "description": "Disable to to disable all subfeatures", "subfeatures": [
+            {"name": "disable_techniques", "enabled": true, "description": "Disable to the remove the ability to disable techniques."},
+            {"name": "manual_color", "enabled": true, "description": "Disable to the remove the ability to assign manual colors to techniques."},
+            {"name": "scoring", "enabled": true, "description": "Disable to the remove the ability to score techniques."},
+            {"name": "comments", "enabled": true, "description": "Disable to the remove the ability to add comments to techniques."},
+            {"name": "clear_annotations", "enabled": true, "description": "Disable to remove the button to clear all annotations on the selected techniques."}
+        ]}
+    ]
+}

salt/playbook/init.sls

@@ -0,0 +1,58 @@
+{% set MASTERIP = salt['pillar.get']('master:mainip', '') %}
+
+playbookdb:
+  file.managed:
+    - name: /opt/so/conf/playbook/redmine.db
+    - source: salt://playbook/files/redmine.db
+    - user: 999
+    - group: 999
+    - makedirs: True
+    - replace: False
+
+playbookwebhook:
+  module.run:
+    - name: sqlite3.modify
+    - db: /opt/so/conf/playbook/redmine.db
+    - sql: "update webhooks set url = 'http://{{MASTERIP}}:7000/playbook/webhook' where project_id = 1"
+
+navigatorconfig:
+  file.managed:
+    - name: /opt/so/conf/playbook/navigator_config.json
+    - source: salt://playbook/files/navigator_config.json
+    - user: 939
+    - group: 939
+    - makedirs: True
+    - template: jinja
+
+so-playbookimage:
+  cmd.run:
+    - name: docker pull --disable-content-trust=false soshybridhunter/so-playbook:HH1.1.1
+
+so-playbook:
+  docker_container.running:
+    - require:
+      - so-playbookimage
+    - image: soshybridhunter/so-playbook:HH1.1.1
+    - hostname: playbook
+    - name: so-playbook
+    - binds:
+      - /opt/so/conf/playbook/redmine.db:/usr/src/redmine/sqlite/redmine.db:rw
+    - port_bindings:
+      - 0.0.0.0:3200:3000
+
+so-navigatorimage:
+  cmd.run:
+    - name: docker pull --disable-content-trust=false soshybridhunter/so-navigator:HH1.1.1
+
+so-navigator:
+  docker_container.running:
+    - require:
+      - so-navigatorimage
+    - image: soshybridhunter/so-navigator:HH1.1.1
+    - hostname: navigator
+    - name: so-navigator
+    - binds:
+      - /opt/so/conf/playbook/navigator_config.json:/nav-app/src/assets/config.json:ro
+      - /opt/so/conf/playbook/nav_layer_playbook.json:/nav-app/src/assets/playbook.json:ro
+    - port_bindings:
+      - 0.0.0.0:4200:4200

salt/sensoroni/init.sls

@@ -29,13 +29,13 @@ sensoronisync:
 
 so-sensoroniimage:
   cmd.run:
-    - name: docker pull --disable-content-trust=false soshybridhunter/so-sensoroni:HH1.1.0
+    - name: docker pull --disable-content-trust=false soshybridhunter/so-sensoroni:HH1.1.1
 
 so-sensoroni:
   docker_container.running:
     - require:
       - so-sensoroniimage
-    - image: soshybridhunter/so-sensoroni:HH1.1.0
+    - image: soshybridhunter/so-sensoroni:HH1.1.1
     - hostname: sensoroni
     - name: so-sensoroni
     - binds:

salt/soctopus/files/SOCtopus.conf

@@ -45,5 +45,9 @@ rtir_creator = root
 slack_url = YOURSLACKWORKSPACE
 slack_webhook = YOURSLACKWEBHOOK
 
+[playbook]
+playbook_url = http://{{ip}}:3200/playbook
+playbook_key = a4a34538782804adfcb8dfae96262514ad70c37c
+
 [log]
 logfile = /tmp/soctopus.log

salt/soctopus/files/templates/generic.template

@@ -0,0 +1,41 @@
+{% set es = salt['pillar.get']('static:masterip', '') %}
+{% set hivehost = salt['pillar.get']('static:masterip', '') %}
+{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
+es_host: {{es}}
+es_port: 9200
+name: Alert-Name
+type: frequency
+index: "*:logstash-*"
+num_events: 1
+timeframe:
+    minutes: 10
+buffer_time:
+    minutes: 10
+allow_buffer_time_overlap: true
+
+filter:
+- query:
+   query_string:
+      query: 'select from test'
+
+alert: modules.so.thehive.TheHiveAlerter
+
+hive_connection:
+  hive_host: https://{{hivehost}}/thehive/
+  hive_apikey: {{hivekey}}
+
+hive_proxies:
+  http: ''
+  https: ''
+
+hive_alert_config:
+  title: '{rule[name]}'
+  type: 'external'
+  source: 'SecurityOnion'
+  description: '`Data:` {match[message]}'
+  severity: 2
+  tags: ['elastalert', 'SecurityOnion']
+  tlp: 3
+  status: 'New'
+  follow: True
+  caseTemplate: '5000'

salt/soctopus/files/templates/osquery.template

@@ -0,0 +1,47 @@
+{% set es = salt['pillar.get']('static:masterip', '') %}
+{% set hivehost = salt['pillar.get']('static:masterip', '') %}
+{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
+es_host: {{es}}
+es_port: 9200
+name: Alert-Name
+type: frequency
+index: "*:logstash-*"
+num_events: 1
+timeframe:
+    minutes: 10
+buffer_time:
+    minutes: 10
+allow_buffer_time_overlap: true
+
+filter:
+- query:
+   query_string:
+      query: 'select from test'
+
+alert: modules.so.thehive.TheHiveAlerter
+
+hive_connection:
+  hive_host: https://{{hivehost}}/thehive/
+  hive_apikey: {{hivekey}}
+
+hive_proxies:
+  http: ''
+  https: ''
+
+hive_alert_config:
+  title: '{rule[name]} -- {match[osquery][hostname]} -- {match[osquery][name]}'
+  type: 'external'
+  source: 'SecurityOnion'
+  description: '`Hostname:` __{match[osquery][hostname]}__  `Live Query:`__[Pivot Link](https://{{es}}/fleet/queries/new?host_uuids={match[osquery][LiveQuery]})__ `Pack:`  __{match[osquery][name]}__ `Data:` {match[osquery][columns]}'
+  severity: 2
+  tags: ['elastalert', 'SecurityOnion']
+  tlp: 3
+  status: 'New'
+  follow: True
+  caseTemplate: '5000'
+
+hive_observable_data_mapping:
+  - ip: '{match[osquery][EndpointIP1]}'
+  - ip: '{match[osquery][EndpointIP2]}'
+  - other: '{match[osquery][hostIdentifier]}'
+  - other: '{match[osquery][hostname]}'

salt/soctopus/init.sls

@@ -13,18 +13,45 @@ soctopussync:
     - group: 939
     - template: jinja
 
+playbookrulesdir:
+  file.directory:
+    - name: /opt/so/rules/elastalert/playbook
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+playbookrulessync:
+  file.recurse:
+    - name: /opt/so/rules/elastalert/playbook
+    - source: salt://soctopus/files/templates
+    - user: 939
+    - group: 939
+    - template: jinja
+
+navigatordefaultlayer:
+  file.managed:
+    - name: /opt/so/conf/playbook/nav_layer_playbook.json
+    - source: salt://playbook/files/nav_layer_playbook.json
+    - user: 939
+    - group: 939
+    - makedirs: True
+    - replace: False
+    - template: jinja
+
 so-soctopusimage:
   cmd.run:
-    - name: docker pull --disable-content-trust=false soshybridhunter/so-soctopus:HH1.1.0
+    - name: docker pull --disable-content-trust=false soshybridhunter/so-soctopus:HH1.1.1
 
 so-soctopus:
   docker_container.running:
     - require:
       - so-soctopusimage
-    - image: soshybridhunter/so-soctopus:HH1.1.0
+    - image: soshybridhunter/so-soctopus:HH1.1.1
     - hostname: soctopus
     - name: so-soctopus
     - binds:
       - /opt/so/conf/soctopus/SOCtopus.conf:/SOCtopus/SOCtopus.conf:ro
+      - /opt/so/rules/elastalert/playbook:/etc/playbook-rules:rw
+      - /opt/so/conf/playbook/nav_layer_playbook.json:/etc/playbook/nav_layer_playbook.json:rw
     - port_bindings:
       - 0.0.0.0:7000:7000

salt/suricata/init.sls

@@ -72,13 +72,13 @@ suriconfigsync:
 
 so-suricataimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false soshybridhunter/so-suricata:HH1.1.0
+   - name: docker pull --disable-content-trust=false soshybridhunter/so-suricata:HH1.1.1
 
 so-suricata:
   docker_container.running:
     - require:
       - so-suricataimage
-    - image: soshybridhunter/so-suricata:HH1.1.0
+    - image: soshybridhunter/so-suricata:HH1.1.1
     - privileged: True
     - environment:
       - INTERFACE={{ interface }}

salt/top.sls

@@ -3,6 +3,7 @@
 {%- set WAZUH = salt['pillar.get']('master:wazuh', '0') -%}
 {%- set GRAFANA = salt['pillar.get']('master:grafana', '0') -%}
 {%- set THEHIVE = salt['pillar.get']('master:thehive', '0') -%}
+{%- set PLAYBOOK = salt['pillar.get']('master:playbook', '0') -%}
 base:
   'G@role:so-sensor':
     - ca
@@ -55,6 +56,10 @@ base:
     {%- if THEHIVE != 0 %}
     - hive
     {%- endif %}
+    {%- if PLAYBOOK != 0 %}
+    - playbook
+    {%- endif %}
+
 
 
   'G@role:so-master':
@@ -87,7 +92,10 @@ base:
     {%- if THEHIVE != 0 %}
     - hive
     {%- endif %}
-
+    {%- if PLAYBOOK != 0 %}
+    - playbook
+    {%- endif %}
+    
 
   # Storage node logic
 

salt/utility/init.sls

@@ -5,6 +5,7 @@
 crossclusterson:
   cmd.script:
     - shell: /bin/bash
+    - cwd: /opt/so
     - runas: socore
     - source: salt://utility/bin/crossthestreams.sh
     - template: jinja
@@ -15,6 +16,7 @@ crossclusterson:
 fixsearch:
   cmd.script:
     - shell: /bin/bash
+    - cwd: /opt/so
     - runas: socore
     - source: salt://utility/bin/eval.sh
     - template: jinja

salt/wazuh/init.sls

@@ -5,7 +5,7 @@ ossecgroup:
   group.present:
     - name: ossec
     - gid: 945
-
+    
 # Add ossecm user
 ossecm:
   user.present:
@@ -13,6 +13,8 @@ ossecm:
     - gid: 945
     - home: /opt/so/wazuh
     - createhome: False
+    - allow_uid_change: True
+    - allow_gid_change: True
 
 # Add ossecr user
 ossecr:
@@ -21,6 +23,8 @@ ossecr:
     - gid: 945
     - home: /opt/so/wazuh
     - createhome: False
+    - allow_uid_change: True
+    - allow_gid_change: True
 
 # Add ossec user
 ossec:
@@ -29,6 +33,8 @@ ossec:
     - gid: 945
     - home: /opt/so/wazuh
     - createhome: False
+    - allow_uid_change: True
+    - allow_gid_change: True
 
 # Add wazuh agent
 wazuhpkgs:

so-setup-network.sh

@@ -61,20 +61,36 @@ add_master_hostfile() {
 }
 
 add_socore_user_master() {
-  echo "Add socore on the master" >> $SETUPLOG 2>&1
+
+  echo "Add socore on the master" >>~/sosetup.log 2>&1
+  # Add user "socore" to the master. This will be for things like accepting keys.
   if [ $OS == 'centos' ]; then
     local ADDUSER=adduser
   else
     local ADDUSER=useradd
   fi
-  # Add user "socore" to the master. This will be for things like accepting keys.
   groupadd --gid 939 socore
   $ADDUSER --uid 939 --gid 939 --home-dir /opt/so socore
-  # Prompt the user to set a password for the user
-  passwd socore
+  # Set the password for socore that we got during setup
+  echo socore:$COREPASS1 | chpasswd --crypt-method=SHA512
 
 }
 
+#add_socore_user_master() {
+#  echo "Add socore on the master" >> $SETUPLOG 2>&1
+#  if [ $OS == 'centos' ]; then
+#    local ADDUSER=adduser
+#  else
+#    local ADDUSER=useradd
+#  fi
+#  # Add user "socore" to the master. This will be for things like accepting keys.
+#  groupadd --gid 939 socore
+#  $ADDUSER --uid 939 --gid 939 --home-dir /opt/so socore
+#  # Prompt the user to set a password for the user
+#  passwd socore
+
+#}
+
 add_socore_user_notmaster() {
   echo "Add socore user on non master" >> $SETUPLOG 2>&1
   # Add socore user to the non master system. Probably not a bad idea to make system user
@@ -168,6 +184,33 @@ checkin_at_boot() {
   echo "startup_states: highstate" >> /etc/salt/minion
 }
 
+check_hive_init_then_reboot() {
+  WAIT_STEP=0
+  MAX_WAIT=100
+    until [ -f /opt/so/state/thehive.txt ] ; do
+    WAIT_STEP=$(( ${WAIT_STEP} + 1 ))
+    echo "Waiting on the_hive to init...Attempt #$WAIT_STEP"
+  	  if [ ${WAIT_STEP} -gt ${MAX_WAIT} ]; then
+  			  echo "ERROR: We waited ${MAX_WAIT} seconds but the_hive is not working."
+  			  exit 5
+  	  fi
+  		  sleep 1s;
+    done
+    docker stop so-thehive
+    docker rm so-thehive
+    shutdown -r now
+}
+
+check_socore_pass() {
+
+  if [ $COREPASS1 == $COREPASS2 ]; then
+    SCMATCH=yes
+  else
+    whiptail_passwords_dont_match
+  fi
+
+}
+
 chown_salt_master() {
 
   echo "Chown the salt dirs on the master for socore" >> $SETUPLOG 2>&1
@@ -246,31 +289,39 @@ copy_ssh_key() {
 
 }
 
-create_bond_nmcli() {
+network_setup() {
   echo "Setting up Bond" >> $SETUPLOG 2>&1
 
   # Set the MTU
-  if [ $NSMSETUP != 'ADVANCED' ]; then
+  if [ "$NSMSETUP" != 'ADVANCED' ]; then
     MTU=1500
   fi
 
-# Create the bond interface
-    nmcli con add ifname bond0 con-name "bond0" type bond mode 0 -- \
-      ipv4.method disabled \
-      ipv6.method link-local \
-      ethernet.mtu $MTU \
-      connection.autoconnect "yes" >> $SETUPLOG 2>&1
+  # Create the bond interface
+  nmcli con add ifname bond0 con-name "bond0" type bond mode 0 -- \
+    ipv4.method disabled \
+    ipv6.method link-local \
+    ethernet.mtu $MTU \
+    connection.autoconnect "yes" >> $SETUPLOG 2>&1
 
-    for BNIC in ${BNICS[@]}; do
-      # Strip the quotes from the NIC names
-      BONDNIC="$(echo -e "${BNIC}" | tr -d '"')"
-      # Create the slave interface and assign it to the bond
-      nmcli con add type ethernet ifname $BONDNIC con-name "bond0-slave-$BONDNIC" master bond0 -- \
-      ethernet.mtu $MTU \
-      connection.autoconnect "yes" >> $SETUPLOG 2>&1
-      # Bring the slave interface up
-      nmcli con up bond0-slave-$BONDNIC >> $SETUPLOG 2>&1
+  for BNIC in ${BNICS[@]}; do
+    # Strip the quotes from the NIC names
+    BONDNIC="$(echo -e "${BNIC}" | tr -d '"')"
+      # Turn off various offloading settings for the interface
+    for i in rx tx sg tso ufo gso gro lro; do
+          ethtool -K $BONDNIC $i off >> $SETUPLOG 2>&1
     done
+    # Create the slave interface and assign it to the bond
+    nmcli con add type ethernet ifname $BONDNIC con-name "bond0-slave-$BONDNIC" master bond0 -- \
+    ethernet.mtu $MTU \
+    connection.autoconnect "yes" >> $SETUPLOG 2>&1
+    # Bring the slave interface up
+    nmcli con up bond0-slave-$BONDNIC >> $SETUPLOG 2>&1
+  done
+  # Replace the variable string in the network script
+  sed -i "s/\$MAININT/${MAININT}/g" ./install_scripts/disable-checksum-offload.sh >> $SETUPLOG 2>&1
+  # Copy the checksum offload script to prevent issues with packet capture
+  cp ./install_scripts/disable-checksum-offload.sh /etc/NetworkManager/dispatcher.d/disable-checksum-offload.sh  >> $SETUPLOG 2>&1
 }
 
 detect_os() {
@@ -458,11 +509,11 @@ install_master() {
 ls_heapsize() {
 
   # Determine LS Heap Size
-  if [ $TOTAL_MEM -ge 16000 ] ; then
-      LS_HEAP_SIZE="4192m"
+  if [ $TOTAL_MEM -ge 32000 ] ; then
+      LS_HEAP_SIZE="1000m"
   else
-      # Set a max of 1GB heap if you have less than 16GB RAM
-      LS_HEAP_SIZE="2g"
+      # If minimal RAM, then set minimal heap
+      LS_HEAP_SIZE="500m"
   fi
 
 }
@@ -477,8 +528,8 @@ master_pillar() {
   echo "  esheap: $ES_HEAP_SIZE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
   echo "  esclustername: {{ grains.host }}" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
   if [ $INSTALLTYPE == 'EVALMODE' ]; then
-    echo "  freq: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
-    echo "  domainstats: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
+    echo "  freq: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
+    echo "  domainstats: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
     echo "  ls_pipeline_batch_size: 125" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
     echo "  ls_input_threads: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
     echo "  ls_batch_count: 125" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
@@ -505,6 +556,7 @@ master_pillar() {
   echo "  osquery: $OSQUERY" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
   echo "  wazuh: $WAZUH" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
   echo "  thehive: $THEHIVE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
+  echo "  playbook: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls
   }
 
 master_static() {
@@ -902,6 +954,15 @@ sensor_pillar() {
 
 }
 
+set_hostname() {
+
+  hostnamectl set-hostname --static $HOSTNAME
+  echo "127.0.0.1   $HOSTNAME $HOSTNAME.localdomain localhost localhost.localdomain localhost4 localhost4.localdomain" > /etc/hosts
+  echo "::1   localhost localhost.localdomain localhost6 localhost6.localdomain6" >> /etc/hosts
+  echo $HOSTNAME > /etc/hostname
+
+}
+
 set_initial_firewall_policy() {
 
   get_main_ip
@@ -1082,6 +1143,26 @@ whiptail_check_exitstatus() {
 
 }
 
+whiptail_create_socore_user() {
+
+  whiptail --title "Security Onion Setup" --msgbox "Set a password for the socore user. This account is used for adding sensors remotely." 8 78
+
+}
+
+whiptail_create_socore_user_password1() {
+
+  COREPASS1=$(whiptail --title "Security Onion Install" --passwordbox \
+  "Enter a password for user socore" 10 60 3>&1 1>&2 2>&3)
+
+}
+
+whiptail_create_socore_user_password2() {
+
+  COREPASS2=$(whiptail --title "Security Onion Install" --passwordbox \
+  "Re-enter a password for user socore" 10 60 3>&1 1>&2 2>&3)
+
+}
+
 whiptail_cur_close_days() {
 
   CURCLOSEDAYS=$(whiptail --title "Security Onion Setup" --inputbox \
@@ -1368,6 +1449,12 @@ whiptail_node_ls_input_batch_count() {
 
 }
 
+whiptail_passwords_dont_match() {
+
+  whiptail --title "Security Onion Setup" --msgbox "Passwords don't match. Please re-enter." 8 78
+
+}
+
 whiptail_rule_setup() {
 
   # Get pulled pork info
@@ -1395,19 +1482,27 @@ whiptail_sensor_config() {
 
 }
 
+whiptail_set_hostname() {
+
+  HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \
+  "Enter the Hostname you would like to set." 10 60 localhost 3>&1 1>&2 2>&3)
+
+  local exitstatus=$?
+  whiptail_check_exitstatus $exitstatus
+
+}
+
 whiptail_setup_complete() {
 
-  whiptail --title "Security Onion Setup" --msgbox "Finished installing this as an $INSTALLTYPE." 8 78
+  whiptail --title "Security Onion Setup" --msgbox "Finished installing this as an $INSTALLTYPE. Press Enter to reboot." 8 78
   install_cleanup
-  exit
 
 }
 
 whiptail_setup_failed() {
 
-  whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $SETUPLOG for details" 8 78
+  whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $SETUPLOG for details. Press Enter to reboot." 8 78
   install_cleanup
-  exit
 
 }
 
@@ -1491,6 +1586,9 @@ if (whiptail_you_sure); then
   # Let folks know they need their management interface already set up.
   whiptail_network_notice
 
+  # Set the hostname to reduce errors
+  whiptail_set_hostname
+
   # Go ahead and gen the keys so we can use them for any sensor type - Disabled for now
   #minio_generate_keys
 
@@ -1540,8 +1638,17 @@ if (whiptail_you_sure); then
       fi
     fi
 
+    whiptail_create_socore_user
+    SCMATCH=no
+    while [ $SCMATCH != yes ]; do
+      whiptail_create_socore_user_password1
+      whiptail_create_socore_user_password2
+      check_socore_pass
+    done
+
     # Last Chance to back out
     whiptail_make_changes
+    set_hostname
     generate_passwords
     auth_pillar
     clear_master
@@ -1555,9 +1662,9 @@ if (whiptail_you_sure); then
     get_main_ip
 
     # Add the user so we can sit back and relax
-    echo ""
-    echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors"
-    echo ""
+    #echo ""
+    #echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors"
+    #echo ""
     add_socore_user_master
 
     # Install salt and dependencies
@@ -1643,8 +1750,14 @@ if (whiptail_you_sure); then
     GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}')
     if [[ $GOODSETUP == '0' ]]; then
       whiptail_setup_complete
+      if [[ $THEHIVE == '1' ]]; then
+        check_hive_init_then_reboot
+      else
+        shutdown -r now
+      fi
     else
       whiptail_setup_failed
+      shutdown -r now
     fi
 
   fi
@@ -1673,6 +1786,7 @@ if (whiptail_you_sure); then
       whiptail_basic_suri
     fi
     whiptail_make_changes
+    set_hostname
     clear_master
     mkdir -p /nsm
     get_filesystem_root
@@ -1683,7 +1797,7 @@ if (whiptail_you_sure); then
       echo -e "XXX\n0\nSetting Initial Firewall Policy... \nXXX"
       set_initial_firewall_policy >> $SETUPLOG 2>&1
       echo -e "XXX\n3\nCreating Bond Interface... \nXXX"
-      create_bond_nmcli >> $SETUPLOG 2>&1
+      network_setup >> $SETUPLOG 2>&1
       echo -e "XXX\n4\nGenerating Sensor Pillar... \nXXX"
       sensor_pillar >> $SETUPLOG 2>&1
       echo -e "XXX\n5\nInstalling Salt Components... \nXXX"
@@ -1716,8 +1830,10 @@ if (whiptail_you_sure); then
     GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}')
     if [[ $GOODSETUP == '0' ]]; then
       whiptail_setup_complete
+      shutdown -r now
     else
       whiptail_setup_failed
+      shutdown -r now
     fi
   fi
 
@@ -1744,7 +1860,7 @@ if (whiptail_you_sure); then
     es_heapsize
     ls_heapsize
     NODE_ES_HEAP_SIZE="600m"
-    NODE_LS_HEAP_SIZE="2000m"
+    NODE_LS_HEAP_SIZE="500m"
     LSPIPELINEWORKERS=1
     LSPIPELINEBATCH=125
     LSINPUTTHREADS=1
@@ -1755,8 +1871,15 @@ if (whiptail_you_sure); then
     BROVERSION=ZEEK
     CURCLOSEDAYS=30
     process_components
+    whiptail_create_socore_user
+    SCMATCH=no
+    while [ $SCMATCH != yes ]; do
+      whiptail_create_socore_user_password1
+      whiptail_create_socore_user_password2
+      check_socore_pass
+    done
     whiptail_make_changes
-    #eval_mode_hostsfile
+    set_hostname
     generate_passwords
     auth_pillar
     clear_master
@@ -1766,14 +1889,11 @@ if (whiptail_you_sure); then
     get_log_size_limit
     get_main_ip
     # Add the user so we can sit back and relax
-    echo ""
-    echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors"
-    echo ""
     add_socore_user_master
     {
       sleep 0.5
       echo -e "XXX\n0\nCreating Bond Interface... \nXXX"
-      create_bond_nmcli >> $SETUPLOG 2>&1
+      network_setup >> $SETUPLOG 2>&1
       echo -e "XXX\n1\nInstalling saltstack... \nXXX"
       saltify >> $SETUPLOG 2>&1
       echo -e "XXX\n3\nInstalling docker... \nXXX"
@@ -1849,6 +1969,7 @@ if (whiptail_you_sure); then
       salt-call state.apply schedule >> $SETUPLOG 2>&1
       salt-call state.apply soctopus >> $SETUPLOG 2>&1
       if [[ $THEHIVE == '1' ]]; then
+        echo -e "XXX\n97\nInstalling The Hive... \nXXX"
         salt-call state.apply hive >> $SETUPLOG 2>&1
       fi
       echo -e "XXX\n98\nSetting checkin to run on boot... \nXXX"
@@ -1861,14 +1982,26 @@ if (whiptail_you_sure); then
     if [ $OS == 'centos' ]; then
       if [[ $GOODSETUP == '1' ]]; then
         whiptail_setup_complete
+        if [[ $THEHIVE == '1' ]]; then
+          check_hive_init_then_reboot
+        else
+          shutdown -r now
+        fi
       else
         whiptail_setup_failed
+        shutdown -r now
       fi
     else
       if [[ $GOODSETUP == '0' ]]; then
         whiptail_setup_complete
+        if [[ $THEHIVE == '1' ]]; then
+          check_hive_init_then_reboot
+        else
+          shutdown -r now
+        fi
       else
         whiptail_setup_failed
+        shutdown -r now
       fi
     fi
   fi
@@ -1905,6 +2038,7 @@ if (whiptail_you_sure); then
       LSINPUTBATCHCOUNT=125
     fi
     whiptail_make_changes
+    set_hostname
     clear_master
     mkdir -p /nsm
     get_filesystem_root
@@ -1946,26 +2080,28 @@ if (whiptail_you_sure); then
     GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}')
     if [[ $GOODSETUP == '0' ]]; then
       whiptail_setup_complete
+      shutdown -r now
     else
       whiptail_setup_failed
+      shutdown -r now
     fi
 
-    set_initial_firewall_policy
-    saltify
-    docker_install
-    configure_minion node
-    set_node_type
-    node_pillar
-    copy_minion_pillar nodes
-    salt_checkin
+    #set_initial_firewall_policy
+    #saltify
+    #docker_install
+    #configure_minion node
+    #set_node_type
+    #node_pillar
+    #copy_minion_pillar nodes
+    #salt_checkin
     # Accept the Salt Key
-    accept_salt_key_remote
+    #accept_salt_key_remote
     # Do the big checkin but first let them know it will take a bit.
-    salt_checkin_message
-    salt_checkin
-    checkin_at_boot
+    #salt_checkin_message
+    #salt_checkin
+    #checkin_at_boot
 
-    whiptail_setup_complete
+    #whiptail_setup_complete
   fi
 
 else