diff --git a/pillar/logstash/fleet.sls b/pillar/logstash/fleet.sls deleted file mode 100644 index fb70e7f0d..000000000 --- a/pillar/logstash/fleet.sls +++ /dev/null @@ -1,6 +0,0 @@ -logstash: - pipelines: - fleet: - config: - - so/0012_input_elastic_agent.conf - - so/9806_output_lumberjack_fleet.conf.jinja \ No newline at end of file diff --git a/pillar/logstash/helix.sls b/pillar/logstash/helix.sls deleted file mode 100644 index ddc1c745b..000000000 --- a/pillar/logstash/helix.sls +++ /dev/null @@ -1,42 +0,0 @@ -logstash: - pipelines: - helix: - config: - - so/0010_input_hhbeats.conf - - so/1033_preprocess_snort.conf - - so/1100_preprocess_bro_conn.conf - - so/1101_preprocess_bro_dhcp.conf - - so/1102_preprocess_bro_dns.conf - - so/1103_preprocess_bro_dpd.conf - - so/1104_preprocess_bro_files.conf - - so/1105_preprocess_bro_ftp.conf - - so/1106_preprocess_bro_http.conf - - so/1107_preprocess_bro_irc.conf - - so/1108_preprocess_bro_kerberos.conf - - so/1109_preprocess_bro_notice.conf - - so/1110_preprocess_bro_rdp.conf - - so/1111_preprocess_bro_signatures.conf - - so/1112_preprocess_bro_smtp.conf - - so/1113_preprocess_bro_snmp.conf - - so/1114_preprocess_bro_software.conf - - so/1115_preprocess_bro_ssh.conf - - so/1116_preprocess_bro_ssl.conf - - so/1117_preprocess_bro_syslog.conf - - so/1118_preprocess_bro_tunnel.conf - - so/1119_preprocess_bro_weird.conf - - so/1121_preprocess_bro_mysql.conf - - so/1122_preprocess_bro_socks.conf - - so/1123_preprocess_bro_x509.conf - - so/1124_preprocess_bro_intel.conf - - so/1125_preprocess_bro_modbus.conf - - so/1126_preprocess_bro_sip.conf - - so/1127_preprocess_bro_radius.conf - - so/1128_preprocess_bro_pe.conf - - so/1129_preprocess_bro_rfb.conf - - so/1130_preprocess_bro_dnp3.conf - - so/1131_preprocess_bro_smb_files.conf - - so/1132_preprocess_bro_smb_mapping.conf - - so/1133_preprocess_bro_ntlm.conf - - so/1134_preprocess_bro_dce_rpc.conf - - so/8001_postprocess_common_ip_augmentation.conf - - so/9997_output_helix.conf.jinja diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls deleted file mode 100644 index cee8eec02..000000000 --- a/pillar/logstash/manager.sls +++ /dev/null @@ -1,8 +0,0 @@ -logstash: - pipelines: - manager: - config: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/0013_input_lumberjack_fleet.conf - - so/9999_output_redis.conf.jinja \ No newline at end of file diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls deleted file mode 100644 index 4d0637dde..000000000 --- a/pillar/logstash/receiver.sls +++ /dev/null @@ -1,8 +0,0 @@ -logstash: - pipelines: - receiver: - config: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/9999_output_redis.conf.jinja - diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls deleted file mode 100644 index 0b660b7ef..000000000 --- a/pillar/logstash/search.sls +++ /dev/null @@ -1,7 +0,0 @@ -logstash: - pipelines: - search: - config: - - so/0900_input_redis.conf.jinja - - so/9805_output_elastic_agent.conf.jinja - - so/9900_output_endgame.conf.jinja diff --git a/pillar/top.sls b/pillar/top.sls index 90b0a41b9..a0fbcb4c1 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -4,6 +4,8 @@ base: - global.adv_global - docker.soc_docker - docker.adv_docker + - firewall.soc_firewall + - firewall.adv_firewall - influxdb.token - logrotate.soc_logrotate - logrotate.adv_logrotate @@ -53,8 +55,6 @@ base: - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - - firewall.soc_firewall - - firewall.adv_firewall - curator.soc_curator - curator.adv_curator - soctopus.soc_soctopus @@ -112,8 +112,6 @@ base: - influxdb.adv_influxdb - backup.soc_backup - backup.adv_backup - - firewall.soc_firewall - - firewall.adv_firewall - zeek.soc_zeek - zeek.adv_zeek - bpf.soc_bpf @@ -164,8 +162,6 @@ base: - curator.adv_curator - backup.soc_backup - backup.adv_backup - - firewall.soc_firewall - - firewall.adv_firewall - zeek.soc_zeek - zeek.adv_zeek - bpf.soc_bpf @@ -258,8 +254,6 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb - - firewall.soc_firewall - - firewall.adv_firewall - zeek.soc_zeek - zeek.adv_zeek - bpf.soc_bpf diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index a14b47e5c..dddab9ddf 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -25,4 +25,5 @@ logstash: pipeline_x_workers: 1 pipeline_x_batch_x_size: 125 pipeline_x_ecs_compatibility: disabled + dmz_nodes: {} diff --git a/salt/logstash/dmz_nodes.yaml b/salt/logstash/dmz_nodes.yaml deleted file mode 100644 index 460088a7d..000000000 --- a/salt/logstash/dmz_nodes.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -# Do not edit this file. Copy it to /opt/so/saltstack/local/salt/logstash/ and make changes there. It should be formatted as a list. -# logstash: -# dmz_nodes: -# - mydmznodehostname1 -# - mydmznodehostname2 -# - mydmznodehostname3 - -logstash: - dmz_nodes: \ No newline at end of file diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index 2a952c754..ca953975f 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -1,226 +1,5 @@ -# Settings file in YAML -# -# Settings can be specified either in hierarchical form, e.g.: -# -# pipeline: -# batch: -# size: 125 -# delay: 5 -# -# Or as flat keys: -# -# pipeline.batch.size: 125 -# pipeline.batch.delay: 5 -# -# ------------ Node identity ------------ -# -# Use a descriptive name for the node: -# -# node.name: test -# -# If omitted the node name will default to the machine's host name -# -# ------------ Data path ------------------ -# -# Which directory should be used by logstash and its plugins -# for any persistent needs. Defaults to LOGSTASH_HOME/data -# -# path.data: -# -# ------------ Pipeline Settings -------------- -# -# The ID of the pipeline. -# -# pipeline.id: main -# -# Set the number of workers that will, in parallel, execute the filters+outputs -# stage of the pipeline. -# -# This defaults to the number of the host's CPU cores. -# -# pipeline.workers: 2 -# -# How many events to retrieve from inputs before sending to filters+workers -# -# pipeline.batch.size: 125 -# -# How long to wait in milliseconds while polling for the next event -# before dispatching an undersized batch to filters+outputs -# -# pipeline.batch.delay: 50 -# -# Force Logstash to exit during shutdown even if there are still inflight -# events in memory. By default, logstash will refuse to quit until all -# received events have been pushed to the outputs. -# -# WARNING: enabling this can lead to data loss during shutdown -# -# pipeline.unsafe_shutdown: false -# -# ------------ Pipeline Configuration Settings -------------- -# -# Where to fetch the pipeline configuration for the main pipeline -# -# path.config: -# /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -# Special Docker path -# path.config: /usr/share/logstash/pipeline - -# -# Pipeline configuration string for the main pipeline -# -# config.string: -# -# At startup, test if the configuration is valid and exit (dry run) -# -# config.test_and_exit: false -# -# Periodically check if the configuration has changed and reload the pipeline -# This can also be triggered manually through the SIGHUP signal -# -# config.reload.automatic: false -# -# How often to check if the pipeline configuration has changed (in seconds) -# -# config.reload.interval: 3s -# -# Show fully compiled configuration as debug log message -# NOTE: --log.level must be 'debug' -# -# config.debug: false -# -# When enabled, process escaped characters such as \n and \" in strings in the -# pipeline configuration files. -# -# config.support_escapes: false -# -# ------------ Module Settings --------------- -# Define modules here. Modules definitions must be defined as an array. -# The simple way to see this is to prepend each `name` with a `-`, and keep -# all associated variables under the `name` they are associated with, and -# above the next, like this: -# -# modules: -# - name: MODULE_NAME -# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE -# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE -# -# Module variable names must be in the format of -# -# var.PLUGIN_TYPE.PLUGIN_NAME.KEY -# -# modules: -# -# ------------ Cloud Settings --------------- -# Define Elastic Cloud settings here. -# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy -# and it may have an label prefix e.g. staging:dXMtZ... -# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' -# cloud.id: -# -# Format of cloud.auth is: : -# This is optional -# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' -# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' -# cloud.auth: elastic: -# -# ------------ Queuing Settings -------------- -# -# Internal queuing model, "memory" for legacy in-memory based queuing and -# "persisted" for disk-based acked queueing. Defaults is memory -# -# queue.type: memory -# -# If using queue.type: persisted, the directory path where the data files will be stored. -# Default is path.data/queue -# -# path.queue: -# -# If using queue.type: persisted, the page data files size. The queue data consists of -# append-only data files separated into pages. Default is 64mb -# -# queue.page_capacity: 64mb -# -# If using queue.type: persisted, the maximum number of unread events in the queue. -# Default is 0 (unlimited) -# -# queue.max_events: 0 -# -# If using queue.type: persisted, the total capacity of the queue in number of bytes. -# If you would like more unacked events to be buffered in Logstash, you can increase the -# capacity using this setting. Please make sure your disk drive has capacity greater than -# the size specified here. If both max_bytes and max_events are specified, Logstash will pick -# whichever criteria is reached first -# Default is 1024mb or 1gb -# -# queue.max_bytes: 1024mb -# -# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint -# Default is 1024, 0 for unlimited -# -# queue.checkpoint.acks: 1024 -# -# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint -# Default is 1024, 0 for unlimited -# -# queue.checkpoint.writes: 1024 -# -# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page -# Default is 1000, 0 for no periodic checkpoint. -# -# queue.checkpoint.interval: 1000 -# -# ------------ Dead-Letter Queue Settings -------------- -# Flag to turn on dead-letter queue. -# -# dead_letter_queue.enable: false - -# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries -# will be dropped if they would increase the size of the dead letter queue beyond this setting. -# Default is 1024mb -# dead_letter_queue.max_bytes: 1024mb - -# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. -# Default is path.data/dead_letter_queue -# -# path.dead_letter_queue: -# -# ------------ Metrics Settings -------------- -# -# Bind address for the metrics REST endpoint -# -# http.host: "127.0.0.1" http.host: 0.0.0.0 -# -# Bind port for the metrics REST endpoint, this option also accept a range -# (9600-9700) and logstash will pick up the first available ports. -# -# http.port: 9600-9700 -# -# ------------ Debugging Settings -------------- -# -# Options for log.level: -# * fatal -# * error -# * warn -# * info (default) -# * debug -# * trace -# -# log.level: info -# path.logs: path.logs: /var/log/logstash -# -# ------------ Other Settings -------------- -# -# Where to find custom plugins -# path.plugins: [] -{% set pipeline_workers = salt['pillar.get']('logstash_settings:ls_pipeline_workers', '1') %} -{% set pipeline_batch = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', '125') %} -{% set pipeline_ecs_compatibility = salt['pillar.get']('logstash_settings:ls_ecs_compatibility', 'disabled') %} - pipeline.workers: {{ pipeline_workers }} pipeline.batch.size: {{ pipeline_batch }} pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }} diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 8e764b0c5..a4d0b87bf 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -36,4 +36,9 @@ logstash: description: Sets ECS compatibility. This is set per pipeline so you should never need to change this. helpLink: logstash.html readonly: True - + dmz_nodes: + description: "List of receiver nodes in DMZs. Prevents sensors from sending to these receivers. Primarily used for external Elastic agents." + helpLink: logstash.html + multiline: True + advanced: True + forcedType: "[]string"