diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 2da8e6c59..10fab2ed1 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -7,6 +7,7 @@ logstash: - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja + - so/9050_output_filebeatmodules.conf.jinja - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja diff --git a/pillar/zeek/init.sls b/pillar/zeek/init.sls index 30a59284a..5eeb273b9 100644 --- a/pillar/zeek/init.sls +++ b/pillar/zeek/init.sls @@ -52,5 +52,4 @@ zeek: - frameworks/signatures/detect-windows-shells redef: - LogAscii::use_json = T; - - LogAscii::json_timestamps = JSON::TS_ISO8601; - - CaptureLoss::watch_interval = 5 mins; \ No newline at end of file + - CaptureLoss::watch_interval = 5 mins; diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index 90eae6b1e..c70701b71 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -17,4 +17,4 @@ . /usr/sbin/so-common -salt-call state.highstate +salt-call state.highstate -linfo diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup new file mode 100755 index 000000000..7a6ae7446 --- /dev/null +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -0,0 +1,67 @@ +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + +#!/bin/bash +# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +default_conf_dir=/opt/so/conf +ELASTICSEARCH_HOST="{{ MYIP }}" +ELASTICSEARCH_PORT=9200 +#ELASTICSEARCH_AUTH="" + +# Define a default directory to load pipelines from +FB_MODULE_YML="/usr/share/filebeat/module-setup.yml" + + +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo +fi +echo "Testing to see if the pipelines are already applied" +ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") +PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) + +if [[ "$PIPELINES" -lt 5 ]]; then + echo "Setting up ingest pipeline(s)" + + for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler + do + echo "Loading $MODULE" + docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML + sleep 2 + done +else + exit 0 +fi + + diff --git a/salt/common/tools/sbin/so-zeek-logs b/salt/common/tools/sbin/so-zeek-logs index 788312df5..f6df7f8aa 100755 --- a/salt/common/tools/sbin/so-zeek-logs +++ b/salt/common/tools/sbin/so-zeek-logs @@ -14,7 +14,6 @@ whiptail_manager_adv_service_zeeklogs() { "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ - "dhcpv6" "DHCP IPv6 Logs" ON \ "dnp3" "DNP3 Logs" ON \ "dns" "DNS Logs" ON \ "dpd" "DPD Logs" ON \ @@ -25,25 +24,20 @@ whiptail_manager_adv_service_zeeklogs() { "irc" "IRC Chat Logs" ON \ "kerberos" "Kerberos Logs" ON \ "modbus" "MODBUS Logs" ON \ - "mqtt" "MQTT Logs" ON \ "notice" "Zeek Notice Logs" ON \ "ntlm" "NTLM Logs" ON \ - "openvpn" "OPENVPN Logs" ON \ "pe" "PE Logs" ON \ "radius" "Radius Logs" ON \ "rfb" "RFB Logs" ON \ "rdp" "RDP Logs" ON \ - "signatures" "Signatures Logs" ON \ "sip" "SIP Logs" ON \ "smb_files" "SMB Files Logs" ON \ "smb_mapping" "SMB Mapping Logs" ON \ "smtp" "SMTP Logs" ON \ "snmp" "SNMP Logs" ON \ - "software" "Software Logs" ON \ "ssh" "SSH Logs" ON \ "ssl" "SSL Logs" ON \ "syslog" "Syslog Logs" ON \ - "telnet" "Telnet Logs" ON \ "tunnel" "Tunnel Logs" ON \ "weird" "Zeek Weird Logs" ON \ "mysql" "MySQL Logs" ON \ diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 45d847698..b5229fca1 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -319,14 +319,6 @@ masterunlock() { fi } -preupgrade_changes_2.3.50_repo() { - # We made repo changes in 2.3.50 and this prepares for that on upgrade - echo "Checking to see if 2.3.50 repo changes are needed." - - [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo - true -} - preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." @@ -517,8 +509,8 @@ up_2.3.2X_to_2.3.30() { INSTALLEDVERSION=2.3.30 } -up_2.3.3X_to_2.3.50_repo() { - echo "Performing 2.3.50 repo actions." +upgrade_to_2.3.50_repo() { + echo "Performing repo changes." if [[ "$OS" == "centos" ]]; then # Import GPG Keys gpg_rpm_import @@ -787,6 +779,12 @@ main() { verify_latest_update_script echo "" + echo "Stopping Salt Master service." + systemctl stop salt-master + echo "" + + upgrade_to_2.3.50_repo + echo "Generating new repo archive" generate_and_clean_tarballs if [ -f /usr/sbin/so-image-common ]; then diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index de6b99000..bf5e7e2b4 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -38,7 +38,7 @@ xpack.security.authc: authz_exception: true {% endif %} node.name: {{ grains.host }} -script.max_compilations_rate: 1000/1m +script.max_compilations_rate: 20000/1m {%- if TRUECLUSTER is sameas true %} {%- if grains.role == 'so-manager' %} {%- if salt['pillar.get']('nodestab', {}) %} diff --git a/salt/elasticsearch/files/ingest/ossec b/salt/elasticsearch/files/ingest/ossec index 868de2798..1c5a0fd42 100644 --- a/salt/elasticsearch/files/ingest/ossec +++ b/salt/elasticsearch/files/ingest/ossec @@ -63,7 +63,8 @@ { "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.dataset", "value": "alert", "override": true } }, + { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.dataset", "value": "ossec.alert", "override": true } }, + { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.kind", "value": "alert", "override": true } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index e5e8560f8..cf2772305 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -53,7 +53,8 @@ { "set": { "if": "ctx.exiftool?.FileDirectory != null", "field": "file.directory", "value": "{{exiftool.FileDirectory}}", "ignore_failure": true }}, { "set": { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem", "value": "{{exiftool.Subsystem}}", "ignore_failure": true }}, { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name", "value": "{{scan.yara.matches.0}}" }}, - { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "dataset", "value": "alert", "override": true }}, + { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "dataset", "value": "strelka.alert", "override": true }}, + { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "event.kind", "value": "alert", "override": true }}, { "rename": { "field": "file.flavors.mime", "target_field": "file.mime_type", "ignore_missing": true }}, { "set": { "if": "ctx.rule?.name != null && ctx.rule?.score == null", "field": "event.severity", "value": 3, "override": true } }, { "convert" : { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}}, diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 214e5c691..3e47fd780 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,5 +1,5 @@ { - "index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], + "index_patterns": ["so-*"], "version":50001, "order":10, "settings":{ @@ -527,11 +527,19 @@ "version":{ "type":"long" } - } - }, + } + }, "x509":{ "type":"object", "dynamic": true + }, + "suricata":{ + "type":"object", + "dynamic": true + }, + "zeek":{ + "type":"object", + "dynamic": true } } } diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 9ae408654..0297d2fe8 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -73,7 +73,13 @@ logging.files: # Set to true to log messages in json format. #logging.json: false + + #========================== Modules configuration ============================ +filebeat.config.modules: + enabled: true + path: ${path.config}/modules.d/*.yml + filebeat.modules: #=========================== Filebeat prospectors ============================= @@ -185,7 +191,6 @@ filebeat.inputs: fields_under_root: true clean_removed: false close_removed: false - {%- if STRELKAENABLED == 1 %} - type: log paths: diff --git a/salt/filebeat/etc/module-setup.yml b/salt/filebeat/etc/module-setup.yml new file mode 100644 index 000000000..431e432b3 --- /dev/null +++ b/salt/filebeat/etc/module-setup.yml @@ -0,0 +1,10 @@ +{%- if grains['role'] in ['so-managersearch', 'so-heavynode', 'so-node'] %} +{%- set MANAGER = salt['grains.get']('host' '') %} +{%- else %} +{%- set MANAGER = salt['grains.get']('master') %} +{%- endif %} + +output.elasticsearch: + enabled: true + hosts: ["https://{{ MANAGER }}:9200"] + ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] diff --git a/salt/filebeat/etc/module_config.yml.jinja b/salt/filebeat/etc/module_config.yml.jinja new file mode 100644 index 000000000..733d47c7e --- /dev/null +++ b/salt/filebeat/etc/module_config.yml.jinja @@ -0,0 +1,18 @@ +# DO NOT EDIT THIS FILE +{%- if MODULES.modules is iterable and MODULES.modules is not string and MODULES.modules|length > 0%} + {%- for module in MODULES.modules.keys() %} +- module: {{ module }} + {%- for fileset in MODULES.modules[module] %} + {{ fileset }}: + enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }} + {#- only manage the settings if the fileset is enabled #} + {%- if MODULES.modules[module][fileset].enabled %} + {%- for var, value in MODULES.modules[module][fileset].items() %} + {%- if var|lower != 'enabled' %} + {{ var }}: {{ value }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endfor %} + {%- endfor %} +{% endif %} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 64cdc47fc..c5d859307 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -20,18 +20,37 @@ {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} +{% set ES_INCLUDED_NODES = ['so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %} + +#only include elastic state for certain nodes +{% if grains.role in ES_INCLUDED_NODES %} +include: + - elasticsearch +{% endif %} + filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc - user: 939 - group: 939 - makedirs: True + +filebeatmoduledir: + file.directory: + - name: /opt/so/conf/filebeat/modules + - user: root + - group: root + - makedirs: True + filebeatlogdir: file.directory: - name: /opt/so/log/filebeat - user: 939 - group: 939 - makedirs: True + filebeatpkidir: file.directory: - name: /opt/so/conf/filebeat/etc/pki @@ -44,6 +63,7 @@ fileregistrydir: - user: 939 - group: 939 - makedirs: True + # This needs to be owned by root filebeatconfsync: file.managed: @@ -55,6 +75,32 @@ filebeatconfsync: - defaults: INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }} OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} + +# Filebeat module config file +filebeatmoduleconfsync: + file.managed: + - name: /opt/so/conf/filebeat/etc/module-setup.yml + - source: salt://filebeat/etc/module-setup.yml + - user: root + - group: root + - template: jinja + +sodefaults_module_conf: + file.managed: + - name: /opt/so/conf/filebeat/modules/securityonion.yml + - source: salt://filebeat/etc/module_config.yml.jinja + - template: jinja + - defaults: + MODULES: {{ SO }} + +thirdparty_module_conf: + file.managed: + - name: /opt/so/conf/filebeat/modules/thirdparty.yml + - source: salt://filebeat/etc/module_config.yml.jinja + - template: jinja + - defaults: + MODULES: {{ THIRDPARTY }} + so-filebeat: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} @@ -65,19 +111,40 @@ so-filebeat: - /nsm:/nsm:ro - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /opt/so/conf/filebeat/etc/module-setup.yml:/usr/share/filebeat/module-setup.yml:ro - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro - /nsm/wazuh/logs/archives:/wazuh/archives:ro + - /opt/so/conf/filebeat/modules:/usr/share/filebeat/modules.d - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro + - /opt/so/log:/logs:ro - port_bindings: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp - 0.0.0.0:5066:5066/tcp +{% for module in THIRDPARTY.modules.keys() %} + {% for submodule in THIRDPARTY.modules[module] %} + {% if THIRDPARTY.modules[module][submodule].enabled and THIRDPARTY.modules[module][submodule]["var.syslog_port"] is defined %} + - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/{{ THIRDPARTY.modules[module][submodule]["var.input"] }} + {% endif %} + {% endfor %} +{% endfor %} - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml +{% if grains.role in ES_INCLUDED_NODES %} +run_module_setup: + cmd.run: + - name: /usr/sbin/so-filebeat-module-setup + - require: + - file: filebeatmoduleconfsync + - docker_container: so-filebeat + - onchanges: + - docker_container: so-elasticsearch +{% endif %} + append_so-filebeat_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja new file mode 100644 index 000000000..6ae6e7cff --- /dev/null +++ b/salt/filebeat/map.jinja @@ -0,0 +1,6 @@ +{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} +{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} + +{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} +{% set SO = SODEFAULTS.securityonion_filebeat %} +{#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml new file mode 100644 index 000000000..be4f81bd1 --- /dev/null +++ b/salt/filebeat/securityoniondefaults.yaml @@ -0,0 +1,31 @@ +{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} +{% set ZEEKLOGLOOKUP = { + 'conn': 'connection', +} %} +securityonion_filebeat: + modules: + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone','so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} + elasticsearch: + server: + enabled: true + var.paths: ["/logs/elasticsearch/*.log"] + logstash: + log: + enabled: true + var.paths: ["/logs/logstash.log"] + {%- endif %} + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} + kibana: + log: + enabled: true + var.paths: ["/logs/kibana/kibana.log"] + {%- endif %} + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %} + redis: + log: + enabled: true + var.paths: ["/logs/redis.log"] + slowlog: + enabled: false + {%- endif %} + \ No newline at end of file diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml new file mode 100644 index 000000000..1b378f84b --- /dev/null +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -0,0 +1,259 @@ +third_party_filebeat: + modules: + aws: + cloudtrail: + enabled: false + cloudwatch: + enabled: false + ec2: + enabled: false + elb: + enabled: false + s3access: + enabled: false + vpcflow: + enabled: false + azure: + activitylogs: + enabled: false + platformlogs: + enabled: false + auditlogs: + enabled: false + signinlogs: + enabled: false + barracuda: + waf: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9503 + spamfirewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9524 + bluecoat: + director: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9505 + cef: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9003 + checkpoint: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9505 + cisco: + asa: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9001 + ftd: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9003 + ios: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9002 + nexus: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9506 + meraki: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9525 + umbrella: + enabled: false + amp: + enabled: false + cyberark: + corepas: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9527 + cylance: + protect: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9508 + f5: + bigipapm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9504 + bigipafm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9528 + fortinet: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9004 + clientendpoint: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9510 + fortimail: + enabled: false + var.input: udp + var.syslog_port: 9350 + gcp: + vpcflow: + enabled: false + firewall: + enabled: false + audit: + enabled: false + google_workspace: + saml: + enabled: false + user_accounts: + enabled: false + login: + enabled: false + admin: + enabled: false + drive: + enabled: false + groups: + enabled: false + imperva: + securesphere: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9511 + infoblox: + nios: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9512 + juniper: + junos: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9513 + netscreen: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9523 + srx: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9006 + microsoft: + defender_atp: + enabled: false + m365_defender: + enabled: false + dhcp: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9515 + misp: + threat: + enabled: false + netflow: + log: + enabled: false + var.netflow_host: 0.0.0.0 + var.netflow_port: 2055 + var.internal_networks: + - private + netscout: + sightline: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9502 + o365: + audit: + enabled: false + okta: + system: + enabled: false + proofpoint: + emailsecurity: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9531 + radware: + defensepro: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9518 + snort: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9532 + snyk: + audit: + enabled: false + vulnerabilities: + enabled: false + sonicwall: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9519 + sophos: + xg: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9005 + utm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9533 + squid: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9520 + tomcat: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9501 + zscaler: + zia: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9521 diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf index a5c1d491c..9ca55b184 100644 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf @@ -3,4 +3,9 @@ input { port => "5044" tags => [ "beat-ext" ] } +} +filter { + mutate { + rename => {"@metadata" => "metadata"} + } } \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja new file mode 100644 index 000000000..20e9f0c0a --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [metadata][pipeline] { + elasticsearch { + id => "filebeat_modules_metadata_pipeline" + pipeline => "%{[metadata][pipeline]}" + hosts => "{{ ES }}" + index => "so-%{[event][module]}-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 62b713f0f..b2a2c15be 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -19,4 +19,4 @@ output { ssl_certificate_verification => false } } -} +} \ No newline at end of file diff --git a/salt/pipeline/load.sls b/salt/pipeline/load.sls new file mode 100644 index 000000000..a43450d0a --- /dev/null +++ b/salt/pipeline/load.sls @@ -0,0 +1,4 @@ +load_elastic_pipelines: + cmd.run: + - name: /usr/sbin/so-filebeat-module-setup + \ No newline at end of file diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index 93295364d..2aaef8e59 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -63,4 +63,4 @@ { "name": "x509", "description": "x.509 grouped by name and issuer", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.issuer"}, { "name": "x509", "description": "x.509 grouped by name and subject", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.subject"}, { "name": "Firewall", "description": "Firewall events grouped by action", "query": "event.dataset:firewall | groupby rule.action"} - ] + ] \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index f3ccc9dda..24be8283c 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -155,6 +155,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_manager and G@saltversion:{{saltversion}}': - match: compound @@ -216,6 +217,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_standalone and G@saltversion:{{saltversion}}': - match: compound @@ -289,6 +291,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -318,6 +321,7 @@ base: {%- endif %} - schedule - docker_clean + - pipeline.load '*_managersearch and G@saltversion:{{saltversion}}': - match: compound @@ -383,6 +387,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_heavynode and G@saltversion:{{saltversion}}': - match: compound @@ -426,6 +431,7 @@ base: {%- endif %} - schedule - docker_clean + - pipeline.load '*_fleet and G@saltversion:{{saltversion}}': - match: compound @@ -471,3 +477,4 @@ base: - zeek - schedule - docker_clean + - pipeline.load diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 02c1cc1ba..ce0d6dccd 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -76,9 +76,9 @@ zeekpolicysync: # Ensure the zeek spool tree (and state.db) ownership is correct zeekspoolownership: file.directory: - - name: /nsm/zeek/spool + - name: /nsm/zeek - user: 937 - - max_depth: 0 + - max_depth: 1 - recurse: - user @@ -183,6 +183,8 @@ so-zeek: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }} - start: {{ START }} - privileged: True + - ulimits: + - core=0 - binds: - /nsm/zeek/logs:/nsm/zeek/logs:rw - /nsm/zeek/spool:/nsm/zeek/spool:rw diff --git a/setup/so-functions b/setup/so-functions index 8b86fc58c..13438b1ba 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2830,7 +2830,6 @@ zeek_logs_enabled() { " - conn"\ " - dce_rpc"\ " - dhcp"\ - " - dhcpv6"\ " - dnp3"\ " - dns"\ " - dpd"\ @@ -2841,25 +2840,20 @@ zeek_logs_enabled() { " - irc"\ " - kerberos"\ " - modbus"\ - " - mqtt"\ " - notice"\ " - ntlm"\ - " - openvpn"\ " - pe"\ " - radius"\ " - rfb"\ " - rdp"\ - " - signatures"\ " - sip"\ " - smb_files"\ " - smb_mapping"\ " - smtp"\ " - snmp"\ - " - software"\ " - ssh"\ " - ssl"\ " - syslog"\ - " - telnet"\ " - tunnel"\ " - weird"\ " - mysql"\ @@ -2871,7 +2865,6 @@ zeek_logs_enabled() { " - conn"\ " - dce_rpc"\ " - dhcp"\ - " - dhcpv6"\ " - dnp3"\ " - dns"\ " - dpd"\ @@ -2882,24 +2875,19 @@ zeek_logs_enabled() { " - irc"\ " - kerberos"\ " - modbus"\ - " - mqtt"\ " - notice"\ " - ntlm"\ - " - openvpn"\ " - pe"\ " - radius"\ " - rfb"\ " - rdp"\ - " - signatures"\ " - sip"\ " - smb_files"\ " - smb_mapping"\ " - smtp"\ " - snmp"\ - " - software"\ " - ssh"\ " - ssl"\ - " - telnet"\ " - tunnel"\ " - weird"\ " - mysql"\ diff --git a/setup/so-whiptail b/setup/so-whiptail index cbf74680f..031265065 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1155,7 +1155,6 @@ whiptail_manager_adv_service_zeeklogs() { "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ - "dhcpv6" "DHCP IPv6 Logs" ON \ "dnp3" "DNP3 Logs" ON \ "dns" "DNS Logs" ON \ "dpd" "DPD Logs" ON \ @@ -1166,25 +1165,20 @@ whiptail_manager_adv_service_zeeklogs() { "irc" "IRC Chat Logs" ON \ "kerberos" "Kerberos Logs" ON \ "modbus" "MODBUS Logs" ON \ - "mqtt" "MQTT Logs" ON \ "notice" "Zeek Notice Logs" ON \ "ntlm" "NTLM Logs" ON \ - "openvpn" "OPENVPN Logs" ON \ "pe" "PE Logs" ON \ "radius" "Radius Logs" ON \ "rfb" "RFB Logs" ON \ "rdp" "RDP Logs" ON \ - "signatures" "Signatures Logs" ON \ "sip" "SIP Logs" ON \ "smb_files" "SMB Files Logs" ON \ "smb_mapping" "SMB Mapping Logs" ON \ "smtp" "SMTP Logs" ON \ "snmp" "SNMP Logs" ON \ - "software" "Software Logs" ON \ "ssh" "SSH Logs" ON \ "ssl" "SSL Logs" ON \ "syslog" "Syslog Logs" ON \ - "telnet" "Telnet Logs" ON \ "tunnel" "Tunnel Logs" ON \ "weird" "Zeek Weird Logs" ON \ "mysql" "MySQL Logs" ON \