Merge pull request #5580 from Security-Onion-Solutions/issue/1257

Issue/1257 - Pillarize ES

pillar/elasticsearch/eval.sls

@@ -1,7 +1,7 @@
 elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
-    - so/so-common-template.json
+    - so/so-common-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja
@@ -10,4 +10,4 @@ elasticsearch:
     - so/so-ossec-template.json.jinja
     - so/so-strelka-template.json.jinja
     - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja
+    - so/so-zeek-template.json.jinja

pillar/elasticsearch/manager.sls

@@ -1,7 +1,7 @@
 elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
-    - so/so-common-template.json
+    - so/so-common-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja

pillar/elasticsearch/search.sls

@@ -1,7 +1,7 @@
 elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
-    - so/so-common-template.json
+    - so/so-common-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja

salt/elasticsearch/config.map.jinja

@@ -0,0 +1,26 @@
+{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+
+{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %}
+  {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %}
+{% endif %}
+
+{% if salt['pillar.get']('elasticsearch:true_cluster', False) %}
+    {% if grains.id.split('_') | last in ['manager','managersearch'] %}
+    {% if salt['pillar.get']('nodestab', {}) %}
+      {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %}
+      {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
+      {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
+        {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %}
+      {% endfor %}
+    {% endif %}
+    {% if grains.id.split('_') | last == 'manager' %}
+      {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': ''}) %}
+    {% endif %}
+  {% else %}
+    {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %}
+    {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %}
+    {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
+  {% endif %}
+{% endif %}
+
+{% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %}

salt/elasticsearch/defaults.yaml

@@ -0,0 +1,62 @@
+elasticsearch:
+  config:
+    node:
+      name: {{ grains.host }}
+      attr:
+        box_type: hot
+    cluster:
+      name: {{ grains.host }}
+      routing:
+        allocation:
+          disk:
+            threshold_enabled: true
+            watermark:
+              low: 95%
+              high: 98%
+              flood_stage: 98%
+    network:
+      host: 0.0.0.0
+    path:
+      logs: /var/log/elasticsearch
+    action:
+      destructive_requires_name: true
+    transport:
+      bind_host: 0.0.0.0
+      publish_host: {{ grains.host }}
+      publish_port: 9300
+    xpack:
+      ml:
+        enabled: false
+      security:
+        enabled: true
+        authc:
+          anonymous:
+            authz_exception: true
+            roles: []
+            username: _anonymous
+        transport:
+          ssl:
+            enabled: true
+            verification_mode: none
+            key: /usr/share/elasticsearch/config/elasticsearch.key
+            certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+            certificate_authorities:
+              - /usr/share/elasticsearch/config/ca.crt
+        http:
+          ssl:
+            enabled: true
+            client_authentication: none
+            key: /usr/share/elasticsearch/config/elasticsearch.key
+            certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+            certificate_authorities: 
+              - /usr/share/elasticsearch/config/ca.crt
+    script:
+      max_compilations_rate: 20000/1m
+    indices:
+      query:
+        bool:
+          max_clause_count: 1500
+
+
+
+

salt/elasticsearch/files/elasticsearch.yaml.jinja

@@ -0,0 +1 @@
+{{ ESCONFIG | yaml(False) }}

salt/elasticsearch/files/elasticsearch.yml

@@ -1,72 +0,0 @@
-{%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %}
-{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip') %}
-{%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
-{%- if TRUECLUSTER is sameas true %}
-  {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name') %}
-{%- else %}
-  {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername') %}
-{%- endif %}
-{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %}
-cluster.name: "{{ ESCLUSTERNAME }}"
-network.host: 0.0.0.0
-path.logs: /var/log/elasticsearch
-action.destructive_requires_name: true
-transport.bind_host: 0.0.0.0
-transport.publish_host: {{ grains.host }}
-transport.publish_port: 9300
-cluster.routing.allocation.disk.threshold_enabled: true
-cluster.routing.allocation.disk.watermark.low: 95%
-cluster.routing.allocation.disk.watermark.high: 98%
-cluster.routing.allocation.disk.watermark.flood_stage: 98%
-xpack.ml.enabled: false
-xpack.security.enabled: true
-xpack.security.transport.ssl.enabled: true
-xpack.security.transport.ssl.verification_mode: none
-xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
-xpack.security.http.ssl.enabled: true
-xpack.security.http.ssl.client_authentication: none
-xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
-{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %}
-xpack.security.authc:
-  anonymous:
-    username: anonymous_user
-    roles: superuser
-    authz_exception: true
-{% endif %}
-node.name: {{ grains.host }}
-script.max_compilations_rate: 20000/1m
-{%- if TRUECLUSTER is sameas true %}
-  {%- if grains.role == 'so-manager' %}
-    {%- if salt['pillar.get']('nodestab', {}) %}
-node.roles: [ master, data, remote_cluster_client ]
-discovery.seed_hosts:
-   - {{ grains.master }}
-      {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-   - {{ SN.split('_')|first }}
-      {%- endfor %}
-    {%- endif %}
-  {%- elif grains.role == 'so-managersearch' %}
-      {%- if salt['pillar.get']('nodestab', {}) %}
-node.roles: [ master, data, remote_cluster_client ]
-discovery.seed_hosts:
-   - {{ grains.master }}
-      {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-   - {{ SN.split('_')|first }}
-      {%- endfor %}
-    {%- endif %}
-node.attr.box_type: {{ NODE_ROUTE_TYPE }}
-  {%- else %}
-node.roles: {{ NODE_ROLES }}
-node.attr.box_type: {{ NODE_ROUTE_TYPE }}
-discovery.seed_hosts:
-   - {{ grains.master }}
-  {%- endif %}
-{%- endif %}
-{%- if TRUECLUSTER is sameas false %}
-node.attr.box_type: {{ NODE_ROUTE_TYPE }}
-{%- endif %}
-indices.query.bool.max_clause_count: 1500

salt/elasticsearch/init.sls

@@ -15,6 +15,8 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
+
+
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
@@ -37,6 +39,7 @@
 {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %}
 {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %}
+{% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %}
 
 
 vm.max_map_count:
@@ -145,9 +148,11 @@ eslog4jfile:
 esyml:
   file.managed:
     - name: /opt/so/conf/elasticsearch/elasticsearch.yml
-    - source: salt://elasticsearch/files/elasticsearch.yml
+    - source: salt://elasticsearch/files/elasticsearch.yaml.jinja
     - user: 930
     - group: 939
+    - defaults:
+        ESCONFIG: {{ ESCONFIG }}
     - template: jinja
 
 #sync templates to /opt/so/conf/elasticsearch/templates

setup/so-functions

@@ -1724,18 +1724,20 @@ manager_global() {
 		"  hot_warm: False"\
 		"elastic:"\
 		"  features: False"\
-		"elasticsearch:"\
+		"elasticsearch:"\ >> "$global_pillar"
-		"  replicas: 0" >> "$global_pillar"
 	if [ -n "$ESCLUSTERNAME" ]; then 
 		printf '%s\n'\
 			"  true_cluster: True"\
-			"  true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar"
+			"  config:"\
+			"    cluster:"\
+			"      name: '$ESCLUSTERNAME'" >> "$global_pillar"
 	else
 		printf '%s\n'\
-			"  true_cluster: False"\
+			"  true_cluster: False" >> "$global_pillar"
-			"  true_cluster_name: 'so'" >> "$global_pillar"
 	fi
+
 	printf '%s\n'\
+		"  replicas: 0"\
 		"  discovery_nodes: 1"\
 		"  hot_warm_enabled: False"\
 		"  cluster_routing_allocation_disk.threshold_enabled: true"\