diff --git a/pillar/elasticsearch/eval.sls b/pillar/elasticsearch/eval.sls index 2dbb08f59..84ff89a23 100644 --- a/pillar/elasticsearch/eval.sls +++ b/pillar/elasticsearch/eval.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja @@ -10,4 +10,4 @@ elasticsearch: - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja - so/so-syslog-template.json.jinja - - so/so-zeek-template.json.jinja \ No newline at end of file + - so/so-zeek-template.json.jinja diff --git a/pillar/elasticsearch/manager.sls b/pillar/elasticsearch/manager.sls index 9ff97de5b..84ff89a23 100644 --- a/pillar/elasticsearch/manager.sls +++ b/pillar/elasticsearch/manager.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja diff --git a/pillar/elasticsearch/search.sls b/pillar/elasticsearch/search.sls index 9ff97de5b..84ff89a23 100644 --- a/pillar/elasticsearch/search.sls +++ b/pillar/elasticsearch/search.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja new file mode 100644 index 000000000..1ca729143 --- /dev/null +++ b/salt/elasticsearch/config.map.jinja @@ -0,0 +1,26 @@ +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} + +{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} + {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} +{% endif %} + +{% if salt['pillar.get']('elasticsearch:true_cluster', False) %} + {% if grains.id.split('_') | last in ['manager','managersearch'] %} + {% if salt['pillar.get']('nodestab', {}) %} + {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} + {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} + {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %} + {% endfor %} + {% endif %} + {% if grains.id.split('_') | last == 'manager' %} + {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': ''}) %} + {% endif %} + {% else %} + {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %} + {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} + {% endif %} +{% endif %} + +{% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml new file mode 100644 index 000000000..e119fb62d --- /dev/null +++ b/salt/elasticsearch/defaults.yaml @@ -0,0 +1,62 @@ +elasticsearch: + config: + node: + name: {{ grains.host }} + attr: + box_type: hot + cluster: + name: {{ grains.host }} + routing: + allocation: + disk: + threshold_enabled: true + watermark: + low: 95% + high: 98% + flood_stage: 98% + network: + host: 0.0.0.0 + path: + logs: /var/log/elasticsearch + action: + destructive_requires_name: true + transport: + bind_host: 0.0.0.0 + publish_host: {{ grains.host }} + publish_port: 9300 + xpack: + ml: + enabled: false + security: + enabled: true + authc: + anonymous: + authz_exception: true + roles: [] + username: _anonymous + transport: + ssl: + enabled: true + verification_mode: none + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt + http: + ssl: + enabled: true + client_authentication: none + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt + script: + max_compilations_rate: 20000/1m + indices: + query: + bool: + max_clause_count: 1500 + + + + diff --git a/salt/elasticsearch/files/elasticsearch.yaml.jinja b/salt/elasticsearch/files/elasticsearch.yaml.jinja new file mode 100644 index 000000000..caf13173f --- /dev/null +++ b/salt/elasticsearch/files/elasticsearch.yaml.jinja @@ -0,0 +1 @@ +{{ ESCONFIG | yaml(False) }} diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml deleted file mode 100644 index 518cd74e9..000000000 --- a/salt/elasticsearch/files/elasticsearch.yml +++ /dev/null @@ -1,72 +0,0 @@ -{%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} -{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip') %} -{%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} -{%- if TRUECLUSTER is sameas true %} - {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name') %} -{%- else %} - {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername') %} -{%- endif %} -{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %} -cluster.name: "{{ ESCLUSTERNAME }}" -network.host: 0.0.0.0 -path.logs: /var/log/elasticsearch -action.destructive_requires_name: true -transport.bind_host: 0.0.0.0 -transport.publish_host: {{ grains.host }} -transport.publish_port: 9300 -cluster.routing.allocation.disk.threshold_enabled: true -cluster.routing.allocation.disk.watermark.low: 95% -cluster.routing.allocation.disk.watermark.high: 98% -cluster.routing.allocation.disk.watermark.flood_stage: 98% -xpack.ml.enabled: false -xpack.security.enabled: true -xpack.security.transport.ssl.enabled: true -xpack.security.transport.ssl.verification_mode: none -xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] -xpack.security.http.ssl.enabled: true -xpack.security.http.ssl.client_authentication: none -xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt -{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} -xpack.security.authc: - anonymous: - username: anonymous_user - roles: superuser - authz_exception: true -{% endif %} -node.name: {{ grains.host }} -script.max_compilations_rate: 20000/1m -{%- if TRUECLUSTER is sameas true %} - {%- if grains.role == 'so-manager' %} - {%- if salt['pillar.get']('nodestab', {}) %} -node.roles: [ master, data, remote_cluster_client ] -discovery.seed_hosts: - - {{ grains.master }} - {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - - {{ SN.split('_')|first }} - {%- endfor %} - {%- endif %} - {%- elif grains.role == 'so-managersearch' %} - {%- if salt['pillar.get']('nodestab', {}) %} -node.roles: [ master, data, remote_cluster_client ] -discovery.seed_hosts: - - {{ grains.master }} - {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - - {{ SN.split('_')|first }} - {%- endfor %} - {%- endif %} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} - {%- else %} -node.roles: {{ NODE_ROLES }} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} -discovery.seed_hosts: - - {{ grains.master }} - {%- endif %} -{%- endif %} -{%- if TRUECLUSTER is sameas false %} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} -{%- endif %} -indices.query.bool.max_clause_count: 1500 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index e4798da4a..9f475c2c3 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,6 +15,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} + + {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} @@ -37,6 +39,7 @@ {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} vm.max_map_count: @@ -145,9 +148,11 @@ eslog4jfile: esyml: file.managed: - name: /opt/so/conf/elasticsearch/elasticsearch.yml - - source: salt://elasticsearch/files/elasticsearch.yml + - source: salt://elasticsearch/files/elasticsearch.yaml.jinja - user: 930 - group: 939 + - defaults: + ESCONFIG: {{ ESCONFIG }} - template: jinja #sync templates to /opt/so/conf/elasticsearch/templates diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json.jinja similarity index 100% rename from salt/elasticsearch/templates/so/so-common-template.json rename to salt/elasticsearch/templates/so/so-common-template.json.jinja diff --git a/setup/so-functions b/setup/so-functions index d851c80b1..06ec7238a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1724,18 +1724,20 @@ manager_global() { " hot_warm: False"\ "elastic:"\ " features: False"\ - "elasticsearch:"\ - " replicas: 0" >> "$global_pillar" + "elasticsearch:"\ >> "$global_pillar" if [ -n "$ESCLUSTERNAME" ]; then printf '%s\n'\ " true_cluster: True"\ - " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" + " config:"\ + " cluster:"\ + " name: '$ESCLUSTERNAME'" >> "$global_pillar" else printf '%s\n'\ - " true_cluster: False"\ - " true_cluster_name: 'so'" >> "$global_pillar" + " true_cluster: False" >> "$global_pillar" fi + printf '%s\n'\ + " replicas: 0"\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\