[wip] Add logscan pipeline

pillar/logstash/search.sls

@@ -13,3 +13,4 @@ logstash:
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
+        - so/9800_ouput_logscan.conf.jinja

salt/elasticsearch/files/ingest/logscan

@@ -0,0 +1,21 @@
+{
+    "description": "logscan",
+    "processors": [
+        { "set": { "target_field": "event.severity", "value": 2 } },
+        { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } },
+        { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } },
+        { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } },
+        { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } },
+        { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } },
+        { "rename": { "field": "source_ip", "target_field": "source.ip" } },
+        { "set": { "if": "model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } },
+        { "set": { "if": "model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } },
+        { "set": { "if": "model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } },
+        { "set": { "if": "model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } },
+        { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } },
+        { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } },
+        { "script": { "lang": "painless", "source": "logscan.attempts.succeeded.amount = logscan.attempts.total.amount - logscan.attempts.failed.amount" , "ignore_failure": true} },
+        { "rename": { "field": "avg_failure_interval", "target_field": "logscan.attempts.failed.avg_interval", "ignore_missing": true } },
+        { "pipeline": { "name": "common" } }
+    ]
+}

salt/filebeat/etc/filebeat.yml

@@ -111,6 +111,19 @@ filebeat.inputs:
       fields: ["source", "prospector", "input", "offset", "beat"]
   fields_under_root: true
 
+- type: log
+  paths:
+    - /opt/so/log/logscan/alerts.log
+  fields:
+    module: logscan
+    dataset: alert
+  processors:
+    - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]
+  fields_under_root: true
+  clean_removed: true
+  close_removed: false
+
 {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor',  'so-helix', 'so-heavynode', 'so-import'] %}
   {%- if ZEEKVER != 'SURICATA' %}
     {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '')  %}

salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja

@@ -0,0 +1,27 @@
+{%- if grains['role'] == 'so-eval' -%}
+{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
+{%- else %}
+{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{%- endif %}
+{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
+{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+
+output {
+  if [module] =~ logscan {
+    elasticsearch {
+      id => "logscan_pipeline"
+      pipeline => "%{module}"
+      hosts => "{{ ES }}"
+    {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
+      user => "{{ ES_USER }}"
+      password => "{{ ES_PASS }}"
+    {% endif %}
+      index => "so-%{[event][module]}"
+      template_name => "so-common"
+      template => "/templates/so-common-template.json"
+      template_overwrite => true
+      ssl => true
+      ssl_certificate_verification => false
+    }
+  }
+}