Merge pull request #5871 from Security-Onion-Solutions/feature/hl_eg

Initial EG stuff

pillar/elasticsearch/manager.sls

@@ -2,6 +2,7 @@ elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json.jinja
+    - so/so-endgame-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja

pillar/elasticsearch/search.sls

@@ -2,6 +2,7 @@ elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json.jinja
+    - so/so-endgame-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja

pillar/logstash/init.sls

@@ -1,6 +1,7 @@
 logstash:
   docker_options:
     port_bindings:
+      - 0.0.0.0:3765:3765
       - 0.0.0.0:5044:5044
       - 0.0.0.0:5644:5644
       - 0.0.0.0:6050:6050

pillar/logstash/manager.sls

@@ -5,5 +5,6 @@ logstash:
       config:
         - so/0009_input_beats.conf      
         - so/0010_input_hhbeats.conf
+        - so/0011_input_endgame.conf
         - so/9999_output_redis.conf.jinja
         

pillar/logstash/search.sls

@@ -14,3 +14,4 @@ logstash:
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
         - so/9800_output_logscan.conf.jinja
+        - so/9900_output_endgame.conf.jinja

salt/curator/files/action/so-endgame-close.yml

@@ -0,0 +1,29 @@
+{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-endgame:close', 30) -%}
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: close
+    description: >-
+      Close Endgame indices older than {{cur_close_days}} days.
+    options:
+      delete_aliases: False
+      timeout_override:
+      continue_if_exception: False
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: regex 
+      value: '^(logstash-endgame.*|so-endgame.*|endgame.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{cur_close_days}}
+      exclude:

salt/curator/files/action/so-endgame-delete.yml

@@ -0,0 +1,29 @@
+{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-endgame:delete', 365) -%}
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete Endgame indices when older than {{ DELETE_DAYS }} days.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: regex
+      value: '^(logstash-endgame.*|so-endgame.*|endgame.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ DELETE_DAYS }}
+      exclude:
+  
+      

salt/curator/files/action/so-endgame-warm.yml

@@ -0,0 +1,24 @@
+{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-endgame:warm', 7) -%}
+actions:
+  1:
+    action: allocation
+    description: "Apply shard allocation filtering rules to the specified indices"
+    options:
+      key: box_type
+      value: warm
+      allocation_type: require
+      wait_for_completion: true
+      timeout_override:
+      continue_if_exception: false
+      disable_action: false
+    filters:
+    - filtertype: pattern
+      kind: regex
+      value: '^(logstash-endgame.*|so-endgame.*|endgame.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ WARM_DAYS }} 
+

salt/logstash/pipelines/config/so/0011_input_endgame.conf

@@ -0,0 +1,12 @@
+input {
+  http {
+    id => "endgame_data"
+    port => 3765
+    codec => es_bulk
+    ssl => true
+    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
+    ssl_certificate => "/usr/share/logstash/filebeat.crt"
+    ssl_key => "/usr/share/logstash/filebeat.key"
+    ssl_verify_mode => "peer"
+  }
+}

salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja

@@ -0,0 +1,29 @@
+{%- if grains['role'] == 'so-eval' -%}
+{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
+{%- else %}
+{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{%- endif %}
+{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
+{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+filter {
+  if [event][module] =~ "endgame" {
+    mutate {
+      remove_field => ["headers", "host"]
+    }
+  }
+}
+output {
+  if [event][module] =~ "endgame" {
+    elasticsearch {
+      id => "endgame_es_output"
+      hosts => "{{ ES }}"
+    {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
+      user => "{{ ES_USER }}"
+      password => "{{ ES_PASS }}"
+    {% endif %}
+      index => "endgame-%{+YYYY.MM.dd}"
+      ssl => true
+      ssl_certificate_verification => false
+    }
+  }
+}