move templates from logstash to elasticsearch

pillar/elasticsearch/search.sls

@@ -0,0 +1,13 @@
+elasticsearch:
+  templates:
+    - so/so-beats-template.json.jinja
+    - so/so-common-template.json
+    - so/so-firewall-template.json.jinja
+    - so/so-flow-template.json.jinja
+    - so/so-ids-template.json.jinja
+    - so/so-import-template.json.jinja
+    - so/so-osquery-template.json.jinja
+    - so/so-ossec-template.json.jinja
+    - so/so-strelka-template.json.jinja
+    - so/so-syslog-template.json.jinja
+    - so/so-zeek-template.json.jinja

pillar/logstash/eval.sls

@@ -1,29 +0,0 @@
-logstash:
-  pipelines:
-    eval:
-      config:
-        - so/0800_input_eval.conf
-        - so/1002_preprocess_json.conf
-        - so/1033_preprocess_snort.conf
-        - so/7100_osquery_wel.conf
-        - so/8999_postprocess_rename_type.conf
-        - so/9000_output_bro.conf.jinja
-        - so/9002_output_import.conf.jinja
-        - so/9033_output_snort.conf.jinja
-        - so/9100_output_osquery.conf.jinja
-        - so/9400_output_suricata.conf.jinja
-        - so/9500_output_beats.conf.jinja
-        - so/9600_output_ossec.conf.jinja
-        - so/9700_output_strelka.conf.jinja
-  templates:
-    - so/so-beats-template.json.jinja
-    - so/so-common-template.json
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/logstash/search.sls

@@ -11,15 +11,3 @@ logstash:
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
-  templates:
-    - so/so-beats-template.json.jinja
-    - so/so-common-template.json
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/top.sls

@@ -11,6 +11,7 @@ base:
     - logstash
     - logstash.manager
     - logstash.search
+    - elasticsearch.search
 
   '*_sensor':
     - static
@@ -41,6 +42,7 @@ base:
     - logstash
     - logstash.manager
     - logstash.search
+    - elasticsearch.search
     - data.*
     - brologs
     - secrets
@@ -75,4 +77,5 @@ base:
     - static
     - logstash
     - logstash.search
+    - elasticsearch.search
     - minions.{{ grains.id }}

salt/common/tools/sbin/so-elasticsearch-templates

@@ -15,13 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-default_salt_dir=/opt/so/saltstack/default
+default_conf_dir=/opt/so/conf
 ELASTICSEARCH_HOST="{{ MANAGERIP}}"
 ELASTICSEARCH_PORT=9200
 #ELASTICSEARCH_AUTH=""
 
 # Define a default directory to load pipelines from
-ELASTICSEARCH_TEMPLATES="$default_salt_dir/salt/logstash/pipelines/templates/so/"
+ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
 
 # Wait for ElasticSearch to initialize
 echo -n "Waiting for ElasticSearch..."

salt/elasticsearch/init.sls

@@ -31,6 +31,8 @@
   {% set esheap = salt['pillar.get']('elasticsearch:esheap', '') %}
 {% endif %}
 
+{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
+
 vm.max_map_count:
   sysctl.present:
     - value: 262144
@@ -63,6 +65,13 @@ esingestdir:
     - group: 939
     - makedirs: True
 
+estemplatedir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates
+    - user: 930
+    - group: 939
+    - makedirs: True
+
 esingestconf:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/ingest
@@ -86,6 +95,36 @@ esyml:
     - group: 939
     - template: jinja
 
+#sync templates to /opt/so/conf/elasticsearch/templates
+{% for TEMPLATE in TEMPLATES %}
+es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
+  file.managed:
+    - source: salt://elasticsearch/templates/{{TEMPLATE}}
+    {% if 'jinja' in TEMPLATE.split('.')[-1] %}
+    - name: /opt/so/conf/elasticsearch/templates/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
+    - template: jinja
+    {% else %}
+    - name: /opt/so/conf/elasticsearch/templates/{{TEMPLATE.split('/')[1]}}
+    {% endif %}
+    - user: 930
+    - group: 939
+{% endfor %}
+
+es_templates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates
+    - source: salt://elasticsearch/templates
+    - user: 930
+    - group: 939
+    - template: jinja
+    - clean: True
+{% if TEMPLATES %}
+    - require:
+  {% for TEMPLATE in TEMPLATES %}
+      - file: es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
+  {% endfor %}
+{% endif %}
+
 nsmesdir:
   file.directory:
     - name: /nsm/elasticsearch

salt/logstash/init.sls

@@ -36,7 +36,6 @@
 {% endif %}
 
 {% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %}
-{% set TEMPLATES = salt['pillar.get']('logstash:templates', {}) %}
 {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
 
 # Create the logstash group
@@ -94,21 +93,6 @@ ls_pipeline_{{PL}}:
 
 {% endfor %}
 
-#sync templates to /opt/so/conf/logstash/etc
-{% for TEMPLATE in TEMPLATES %}
-ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
-  file.managed:
-    - source: salt://logstash/pipelines/templates/{{TEMPLATE}}
-    {% if 'jinja' in TEMPLATE.split('.')[-1] %}
-    - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
-    - template: jinja
-    {% else %}
-    - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1]}}
-    {% endif %}
-    - user: 931
-    - group: 939
-{% endfor %}
-
 lspipelinesyml:
   file.managed:
     - name: /opt/so/conf/logstash/etc/pipelines.yml
@@ -126,12 +110,6 @@ lsetcsync:
     - group: 939
     - template: jinja
     - clean: True
-{% if TEMPLATES %}
-    - require:
-  {% for TEMPLATE in TEMPLATES %}
-      - file: ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
-  {% endfor %}
-{% endif %}
     - exclude_pat: pipelines*
 
 # Create the import directory
@@ -171,13 +149,7 @@ so-logstash:
       - {{ BINDING }}
 {% endfor %}
     - binds:
-{% for TEMPLATE in TEMPLATES %}
-  {% if 'jinja' in TEMPLATE.split('.')[-1] %}
-      - /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}:/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}:ro
-  {% else %}
-      - /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1]}}:/{{TEMPLATE.split('/')[1]}}:ro
-  {% endif %}
-{% endfor %}
+      - /opt/so/conf/elasticsearch/templates/:/templates/:ro
       - /opt/so/conf/logstash/etc/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro
       - /opt/so/conf/logstash/etc/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
       - /opt/so/conf/logstash/etc/pipelines.yml:/usr/share/logstash/config/pipelines.yml
@@ -206,7 +178,4 @@ so-logstash:
       - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
   {% endfor %}
 {% endfor %}
-{% for TEMPLATE in TEMPLATES %}
-      - file: ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
-{% endfor %}
-#     - file: /opt/so/conf/logstash/rulesets
+      - file: /opt/so/conf/elasticsearch/templates/*

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-zeek-%{+YYYY.MM.dd}"
       template_name => "so-zeek"
-      template => "/so-zeek-template.json"
+      template => "/templates/so-zeek-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-import-%{+YYYY.MM.dd}"
       template_name => "so-import"
-      template => "/so-import-template.json"
+      template => "/templates/so-import-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja

@@ -9,7 +9,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-flow-%{+YYYY.MM.dd}"
       template_name => "so-flow"
-      template => "/so-flow-template.json"
+      template => "/templates/so-flow-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja

@@ -9,7 +9,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
       template_name => "so-ids"
-      template => "/so-ids-template.json"
+      template => "/templates/so-ids-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-syslog-%{+YYYY.MM.dd}"
       template_name => "so-syslog"
-      template => "/so-syslog-template.json"
+      template => "/templates/so-syslog-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-osquery-%{+YYYY.MM.dd}"
       template_name => "so-osquery"
-      template => "/so-osquery-template.json"
+      template => "/templates/so-osquery-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -9,7 +9,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-firewall-%{+YYYY.MM.dd}"
       template_name => "so-firewall"
-      template => "/so-firewall-template.json"
+      template => "/templates/so-firewall-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
       template_name => "so-ids" 
-      template => "/so-ids-template.json"
+      template => "/templates/so-ids-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-beats-%{+YYYY.MM.dd}"
       template_name => "so-beats"
-      template => "/so-beats-template.json"
+      template => "/templates/so-beats-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-ossec-%{+YYYY.MM.dd}"
       template_name => "so-ossec"
-      template => "/so-ossec-template.json"
+      template => "/templates/so-ossec-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -10,7 +10,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-strelka-%{+YYYY.MM.dd}"
       template_name => "so-strelka"
-      template => "/so-strelka-template.json"
+      template => "/templates/so-strelka-template.json"
       template_overwrite => true
     }
   }