Merge remote-tracking branch 'origin/2.4/dev' into kilo

salt/elasticsearch/config.sls

@@ -118,6 +118,19 @@ esingestconf:
     - user: 930
     - group: 939
 
+# Auto-generate Elasticsearch ingest node pipelines from pillar
+{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
+es_ingest_conf_{{pipeline}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }}
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ config }}
+    - template: jinja
+    - onchanges_in:
+      - file: so-pipelines-reload
+{%     endfor %}
+
 eslog4jfile:
   file.managed:
     - name: /opt/so/conf/elasticsearch/log4j2.properties

salt/elasticsearch/defaults.yaml

@@ -55,6 +55,87 @@ elasticsearch:
             key: /usr/share/elasticsearch/config/elasticsearch.key
             verification_mode: none
   enabled: false
+  pipelines:
+    custom001:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom001
+      - pipeline:
+          name: common
+    custom002:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom002
+      - pipeline:
+          name: common
+    custom003:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom003
+      - pipeline:
+          name: common
+    custom004:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom004
+      - pipeline:
+          name: common
+    custom005:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom005
+      - pipeline:
+          name: common
+    custom006:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom006
+      - pipeline:
+          name: common
+    custom007:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom007
+      - pipeline:
+          name: common
+    custom008:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom008
+      - pipeline:
+          name: common
+    custom009:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom009
+      - pipeline:
+          name: common
+    custom010:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom010
+      - pipeline:
+          name: common
   index_settings:
     global_overrides:
       index_template:

salt/elasticsearch/soc_elasticsearch.yaml

@@ -45,6 +45,28 @@ elasticsearch:
             description: Max number of boolean clauses per query.
             global: True
             helpLink: elasticsearch.html
+  pipelines:
+    custom001: &pipelines
+      description:
+        description: Description of the ingest node pipeline
+        global: True
+        advanced: True
+        helpLink: elasticsearch.html
+      processors:
+        description: Processors for the ingest node pipeline
+        global: True
+        advanced: True
+        multiline: True
+        helpLink: elasticsearch.html
+    custom002: *pipelines
+    custom003: *pipelines
+    custom004: *pipelines
+    custom005: *pipelines
+    custom006: *pipelines
+    custom007: *pipelines
+    custom008: *pipelines
+    custom009: *pipelines
+    custom010: *pipelines
   index_settings:
     global_overrides:
       index_template: