Merge pull request #9654 from Security-Onion-Solutions/fix/logstash_cleanup

FIX: Logstash Pipeline Cleanup

pillar/logstash/manager.sls

@@ -2,8 +2,6 @@ logstash:
   pipelines:
     manager:
       config:
-        - so/0009_input_beats.conf      
-        - so/0010_input_hhbeats.conf
         - so/0011_input_endgame.conf
         - so/0012_input_elastic_agent.conf     
         - so/9999_output_redis.conf.jinja

pillar/logstash/receiver.sls

@@ -2,8 +2,7 @@ logstash:
   pipelines:
     receiver:
       config:
-        - so/0009_input_beats.conf      
-        - so/0010_input_hhbeats.conf
         - so/0011_input_endgame.conf
+        - so/0012_input_elastic_agent.conf
         - so/9999_output_redis.conf.jinja
         

pillar/logstash/search.sls

@@ -3,16 +3,5 @@ logstash:
     search:
       config:
         - so/0900_input_redis.conf.jinja
-        - so/9000_output_zeek.conf.jinja
-        - so/9002_output_import.conf.jinja
-        - so/9034_output_syslog.conf.jinja
-        - so/9050_output_filebeatmodules.conf.jinja
-        - so/9100_output_osquery.conf.jinja  
-        - so/9400_output_suricata.conf.jinja
-        - so/9500_output_beats.conf.jinja
-        - so/9600_output_ossec.conf.jinja
-        - so/9700_output_strelka.conf.jinja
-        - so/9800_output_logscan.conf.jinja
-        - so/9801_output_rita.conf.jinja
         - so/9805_output_elastic_agent.conf.jinja
         - so/9900_output_endgame.conf.jinja

salt/logstash/pipelines/config/so/0009_input_beats.conf

@@ -1,11 +0,0 @@
-input {
-  beats {
-    port => "5044"
-    tags => [ "beat-ext" ]
-  }
-}
-filter {
-  mutate {
-    rename => {"@metadata" => "metadata"}
-  }
-}

salt/logstash/pipelines/config/so/0010_input_hhbeats.conf

@@ -1,40 +0,0 @@
-input {
-  beats {
-    port => "5644"
-    ssl => true
-    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
-    ssl_certificate => "/usr/share/logstash/filebeat.crt"
-    ssl_key => "/usr/share/logstash/filebeat.key"
-    #tags => [ "beat" ]
-  }
-}
-filter {
-  if [type] == "ids" or [type] =~ "bro" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_field => { "sensor_name" => "%{[beat][name]}" }
-      add_field => { "syslog-host_from" => "%{[beat][name]}" }
-      remove_field => [ "beat", "prospector", "input", "offset" ]
-    }
-  }
-  if [type] =~ "ossec" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_field => { "syslog-host_from" => "%{[beat][name]}" }
-      remove_field => [ "beat", "prospector", "input", "offset" ]
-    }
-   }
-    if [type] == "osquery" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_tag => ["osquery"]
-        	}
-   json {
-    source => "message"
-    target => "osquery"
-        }
-  }
-}

salt/logstash/pipelines/config/so/0800_input_eval.conf

@@ -1,204 +0,0 @@
-# Updated by: Mike Reeves
-# Last Update: 11/1/2018
-
-input {
-  file {
-	  path => "/suricata/eve.json"
-		type => "ids"
-                add_field => { "engine" => "suricata" }
-	}
-	file {
-		path => "/nsm/zeek/logs/current/conn*.log"
-		type => "zeek.conn"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dce_rpc*.log"
-		type => "zeek.dce_rpc"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dhcp*.log"
-		type => "zeek.dhcp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dnp3*.log"
-		type => "zeek.dnp3"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dns*.log"
-		type => "zeek.dns"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dpd*.log"
-		type => "zeek.dpd"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/files*.log"
-		type => "zeek.files"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ftp*.log"
-		type => "zeek.ftp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/http*.log"
-		type => "zeek.http"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/intel*.log"
-		type => "zeek.intel"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/irc*.log"
-		type => "zeek.irc"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/kerberos*.log"
-		type => "zeek.kerberos"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/modbus*.log"
-		type => "zeek.modbus"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/mysql*.log"
-		type => "zeek.mysql"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/notice*.log"
-		type => "zeek.notice"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ntlm*.log"
-		type => "zeek.ntlm"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/pe*.log"
-		type => "zeek.pe"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/radius*.log"
-		type => "zeek.radius"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/rdp*.log"
-		type => "zeek.rdp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/rfb*.log"
-		type => "zeek.rfb"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/signatures*.log"
-		type => "zeek.signatures"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/sip*.log"
-		type => "zeek.sip"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/smb_files*.log"
-		type => "zeek.smb_files"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/smb_mapping*.log"
-		type => "zeek.smb_mapping"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/smtp*.log"
-		type => "zeek.smtp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/snmp*.log"
-		type => "zeek.snmp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/socks*.log"
-		type => "zeek.socks"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/software*.log"
-		type => "zeek.software"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ssh*.log"
-		type => "zeek.ssh"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ssl*.log"
-		type => "zeek.ssl"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/syslog*.log"
-		type => "zeek.syslog"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/tunnel*.log"
-		type => "zeek.tunnels"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/weird*.log"
-		type => "zeek.weird"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/x509*.log"
-		type => "zeek.x509"
-	    	tags => ["zeek"]
-	}
-  file {
-    path => "/wazuh/alerts/alerts.json"
-    type => "ossec"
-  }
-#  file {
-#    path => "/wazuh/archives/archives.json"
-#    type => "ossec_archive"
-#  }
-  file {
-    path => "/osquery/logs/result.log"
-    type => "osquery"
-    tags => ["osquery"]
-  }
-  file {
-    path => "/strelka/strelka.log"
-    type => "strelka"
-  }
-}
-filter {
-  if "import" in [tags] {
-	mutate {
-	  #add_tag => [ "conf_file_0007"]
-	}
-  }
-}

salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja

@@ -1,23 +0,0 @@
-{%- if grains.role == 'so-heavynode' %}
-{%- set HOST = GLOBALS.hostname %}
-{%- else %}
-{%- set HOST = GLOBALS.manager %}
-{% endif -%}
-{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
-{%- set access_key = salt['pillar.get']('minio:access_key', '') %}
-{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %}
-{%- set INTERVAL = salt['pillar.get']('s3_settings:interval', 5) %}
-input {
-	s3 {
-        access_key_id => "{{ access_key }}"
-        secret_access_key => "{{ access_secret }}"
-        endpoint => "https://{{ HOST }}:9595"
-        bucket => "logstash"
-        delete => true
-        interval => {{ INTERVAL }}
-        codec => json
-        additional_settings => {
-            "force_path_style" => true
-            }
-    }
-}

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "zeek" and "import" not in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-zeek"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if "import" in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-import"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "syslog" {
-    elasticsearch {
-      pipeline => "%{module}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-syslog"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja

@@ -1,14 +0,0 @@
-output {
-  if "filebeat" in [metadata][pipeline] {
-    elasticsearch {
-      id => "filebeat_modules_metadata_pipeline"
-      pipeline => "%{[metadata][pipeline]}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-%{[event][module]}-%{+YYYY.MM.dd}"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "osquery" and "live_query" not in [dataset] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}" 
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-osquery"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -1,12 +0,0 @@
-output {
-  if [dataset] =~ "firewall" {
-    elasticsearch {
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-firewall"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "suricata" and "import" not in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-ids"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -1,26 +0,0 @@
-output {
-  if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] {
-    if [metadata][_id] {
-      elasticsearch {
-        pipeline => "beats.common" 
-        hosts => "{{ GLOBALS.manager }}"
-        user => "{{ ES_USER }}"
-        password => "{{ ES_PASS }}"
-        index => "so-beats"
-        ssl => true
-        ssl_certificate_verification => false
-        document_id => "%{[metadata][_id]}" 
-      }
-      } else {
-      elasticsearch {
-        pipeline => "beats.common" 
-        hosts => "{{ GLOBALS.manager }}"
-        user => "{{ ES_USER }}"
-        password => "{{ ES_PASS }}"
-        index => "so-beats"
-        ssl => true
-        ssl_certificate_verification => false
-      }
-  }
- }
-}

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "ossec" {
-    elasticsearch {
-      pipeline => "%{module}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-ossec"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -1,14 +0,0 @@
-output {
-  if [module] =~ "strelka" {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}" 
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-strelka"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}
-

salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja

@@ -1,14 +0,0 @@
-output {
-  if [module] =~ "logscan" {
-    elasticsearch {
-      id => "logscan_pipeline"
-      pipeline => "logscan.alert"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-logscan"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "rita" and "import" not in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-rita"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}