From b3123f7895951675411150ce3d172120a0005363 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 17:57:07 +0000 Subject: [PATCH 1/2] Remove unnecessary Logstash pipelines from the pillar --- pillar/logstash/manager.sls | 4 +--- pillar/logstash/receiver.sls | 5 ++--- pillar/logstash/search.sls | 11 ----------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index cfeb0a6ae..41a2197fd 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -2,9 +2,7 @@ logstash: pipelines: manager: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls index 09c2549e6..4d0637dde 100644 --- a/pillar/logstash/receiver.sls +++ b/pillar/logstash/receiver.sls @@ -2,8 +2,7 @@ logstash: pipelines: receiver: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index fb10d18e7..0b660b7ef 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -3,16 +3,5 @@ logstash: search: config: - so/0900_input_redis.conf.jinja - - so/9000_output_zeek.conf.jinja - - so/9002_output_import.conf.jinja - - so/9034_output_syslog.conf.jinja - - so/9050_output_filebeatmodules.conf.jinja - - so/9100_output_osquery.conf.jinja - - so/9400_output_suricata.conf.jinja - - so/9500_output_beats.conf.jinja - - so/9600_output_ossec.conf.jinja - - so/9700_output_strelka.conf.jinja - - so/9800_output_logscan.conf.jinja - - so/9801_output_rita.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja From e4271043c6b7aae64aa92f20a7d0c7c552026aae Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 18:05:14 +0000 Subject: [PATCH 2/2] Remove unnecessary Logstash pipelines --- .../pipelines/config/so/0009_input_beats.conf | 11 - .../config/so/0010_input_hhbeats.conf | 40 ---- .../pipelines/config/so/0800_input_eval.conf | 204 ------------------ .../config/so/0899_input_minio.conf.jinja | 23 -- .../config/so/9000_output_zeek.conf.jinja | 13 -- .../config/so/9002_output_import.conf.jinja | 13 -- .../config/so/9034_output_syslog.conf.jinja | 13 -- .../so/9050_output_filebeatmodules.conf.jinja | 14 -- .../config/so/9100_output_osquery.conf.jinja | 13 -- .../config/so/9200_output_firewall.conf.jinja | 12 -- .../config/so/9400_output_suricata.conf.jinja | 13 -- .../config/so/9500_output_beats.conf.jinja | 26 --- .../config/so/9600_output_ossec.conf.jinja | 13 -- .../config/so/9700_output_strelka.conf.jinja | 14 -- .../config/so/9800_output_logscan.conf.jinja | 14 -- .../config/so/9801_output_rita.conf.jinja | 13 -- 16 files changed, 449 deletions(-) delete mode 100644 salt/logstash/pipelines/config/so/0009_input_beats.conf delete mode 100644 salt/logstash/pipelines/config/so/0010_input_hhbeats.conf delete mode 100644 salt/logstash/pipelines/config/so/0800_input_eval.conf delete mode 100644 salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9002_output_import.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf deleted file mode 100644 index 8643a64b4..000000000 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ /dev/null @@ -1,11 +0,0 @@ -input { - beats { - port => "5044" - tags => [ "beat-ext" ] - } -} -filter { - mutate { - rename => {"@metadata" => "metadata"} - } -} diff --git a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf b/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf deleted file mode 100644 index 050d01d73..000000000 --- a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf +++ /dev/null @@ -1,40 +0,0 @@ -input { - beats { - port => "5644" - ssl => true - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - #tags => [ "beat" ] - } -} -filter { - if [type] == "ids" or [type] =~ "bro" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "sensor_name" => "%{[beat][name]}" } - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] =~ "ossec" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] == "osquery" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_tag => ["osquery"] - } - json { - source => "message" - target => "osquery" - } - } -} diff --git a/salt/logstash/pipelines/config/so/0800_input_eval.conf b/salt/logstash/pipelines/config/so/0800_input_eval.conf deleted file mode 100644 index 35a977d04..000000000 --- a/salt/logstash/pipelines/config/so/0800_input_eval.conf +++ /dev/null @@ -1,204 +0,0 @@ -# Updated by: Mike Reeves -# Last Update: 11/1/2018 - -input { - file { - path => "/suricata/eve.json" - type => "ids" - add_field => { "engine" => "suricata" } - } - file { - path => "/nsm/zeek/logs/current/conn*.log" - type => "zeek.conn" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dce_rpc*.log" - type => "zeek.dce_rpc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dhcp*.log" - type => "zeek.dhcp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dnp3*.log" - type => "zeek.dnp3" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dns*.log" - type => "zeek.dns" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dpd*.log" - type => "zeek.dpd" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/files*.log" - type => "zeek.files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ftp*.log" - type => "zeek.ftp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/http*.log" - type => "zeek.http" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/intel*.log" - type => "zeek.intel" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/irc*.log" - type => "zeek.irc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/kerberos*.log" - type => "zeek.kerberos" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/modbus*.log" - type => "zeek.modbus" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/mysql*.log" - type => "zeek.mysql" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/notice*.log" - type => "zeek.notice" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ntlm*.log" - type => "zeek.ntlm" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/pe*.log" - type => "zeek.pe" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/radius*.log" - type => "zeek.radius" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rdp*.log" - type => "zeek.rdp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rfb*.log" - type => "zeek.rfb" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/signatures*.log" - type => "zeek.signatures" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/sip*.log" - type => "zeek.sip" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_files*.log" - type => "zeek.smb_files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_mapping*.log" - type => "zeek.smb_mapping" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smtp*.log" - type => "zeek.smtp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/snmp*.log" - type => "zeek.snmp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/socks*.log" - type => "zeek.socks" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/software*.log" - type => "zeek.software" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssh*.log" - type => "zeek.ssh" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssl*.log" - type => "zeek.ssl" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/syslog*.log" - type => "zeek.syslog" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/tunnel*.log" - type => "zeek.tunnels" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/weird*.log" - type => "zeek.weird" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/x509*.log" - type => "zeek.x509" - tags => ["zeek"] - } - file { - path => "/wazuh/alerts/alerts.json" - type => "ossec" - } -# file { -# path => "/wazuh/archives/archives.json" -# type => "ossec_archive" -# } - file { - path => "/osquery/logs/result.log" - type => "osquery" - tags => ["osquery"] - } - file { - path => "/strelka/strelka.log" - type => "strelka" - } -} -filter { - if "import" in [tags] { - mutate { - #add_tag => [ "conf_file_0007"] - } - } -} diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja deleted file mode 100644 index 7a0848b39..000000000 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ /dev/null @@ -1,23 +0,0 @@ -{%- if grains.role == 'so-heavynode' %} -{%- set HOST = GLOBALS.hostname %} -{%- else %} -{%- set HOST = GLOBALS.manager %} -{% endif -%} -{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{%- set access_key = salt['pillar.get']('minio:access_key', '') %} -{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} -{%- set INTERVAL = salt['pillar.get']('s3_settings:interval', 5) %} -input { - s3 { - access_key_id => "{{ access_key }}" - secret_access_key => "{{ access_secret }}" - endpoint => "https://{{ HOST }}:9595" - bucket => "logstash" - delete => true - interval => {{ INTERVAL }} - codec => json - additional_settings => { - "force_path_style" => true - } - } -} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja deleted file mode 100644 index 7b8c03f45..000000000 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "zeek" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-zeek" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja deleted file mode 100644 index a57830229..000000000 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if "import" in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-import" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja deleted file mode 100644 index 4c49c61ea..000000000 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "syslog" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-syslog" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja deleted file mode 100644 index 672a83876..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if "filebeat" in [metadata][pipeline] { - elasticsearch { - id => "filebeat_modules_metadata_pipeline" - pipeline => "%{[metadata][pipeline]}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-%{[event][module]}-%{+YYYY.MM.dd}" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja deleted file mode 100644 index 8dbea872e..000000000 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "osquery" and "live_query" not in [dataset] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-osquery" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja deleted file mode 100644 index 7942aa50c..000000000 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ /dev/null @@ -1,12 +0,0 @@ -output { - if [dataset] =~ "firewall" { - elasticsearch { - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-firewall" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja deleted file mode 100644 index 13df33e16..000000000 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "suricata" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ids" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja deleted file mode 100644 index b4aafecad..000000000 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -output { - if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] { - if [metadata][_id] { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - document_id => "%{[metadata][_id]}" - } - } else { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - } - } - } -} diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja deleted file mode 100644 index ca3eeb6c1..000000000 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "ossec" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ossec" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja deleted file mode 100644 index 281cdda5b..000000000 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "strelka" { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-strelka" - ssl => true - ssl_certificate_verification => false - } - } -} - diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja deleted file mode 100644 index 8127de23a..000000000 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "logscan" { - elasticsearch { - id => "logscan_pipeline" - pipeline => "logscan.alert" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-logscan" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja deleted file mode 100644 index 7f9d795e6..000000000 --- a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "rita" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-rita" - ssl => true - ssl_certificate_verification => false - } - } -}