diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls new file mode 100644 index 000000000..7a8a6675c --- /dev/null +++ b/salt/elasticsearch/cluster.sls @@ -0,0 +1,164 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} +{% if GLOBALS.role != 'so-heavynode' %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% endif %} + +escomponenttemplates: + file.recurse: + - name: /opt/so/conf/elasticsearch/templates/component + - source: salt://elasticsearch/templates/component + - user: 930 + - group: 939 + - clean: True + - onchanges_in: + - file: so-elasticsearch-templates-reload + - show_changes: False + +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory +so_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/index + - clean: True + {%- if SO_MANAGED_INDICES %} + - require: + {%- for index in SO_MANAGED_INDICES %} + - file: so_index_template_{{index}} + {%- endfor %} + {%- endif %} + +# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) +# These index templates are for the core SO datasets and are always required +{% for index, settings in ES_INDEX_SETTINGS.items() %} +{% if settings.index_template is defined %} +so_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja + - onchanges_in: + - file: so-elasticsearch-templates-reload +{% endif %} +{% endfor %} + +{% if GLOBALS.role != "so-heavynode" %} +# Auto-generate optional index templates for integration | input | content packages +# These index templates are not used by default (until user adds package to an agent policy). +# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} +addon_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja + - show_changes: False + - onchanges_in: + - file: addon-elasticsearch-templates-reload +{% endif %} +{% endfor %} +{% endif %} + +{% if GLOBALS.role in GLOBALS.manager_roles %} +so-es-cluster-settings: + cmd.run: + - name: /usr/sbin/so-elasticsearch-cluster-settings + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + - http: wait_for_so-elasticsearch +{% endif %} + +# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) +so-elasticsearch-ilm-policy-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-ilm-policy-load + - cwd: /opt/so + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-ilm-policy-load-script + - onchanges: + - file: so-elasticsearch-ilm-policy-load-script + +so-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/estemplates.txt + +addon-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/addon_estemplates.txt + +# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script +so-elasticsearch-templates: + cmd.run: +{%- if GLOBALS.role == "so-heavynode" %} + - name: /usr/sbin/so-elasticsearch-templates-load --heavynode +{%- else %} + - name: /usr/sbin/so-elasticsearch-templates-load +{%- endif %} + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +so-elasticsearch-pipelines: + cmd.run: + - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-pipelines-script + +so-elasticsearch-roles-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-roles-load + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %} +{% set ap = "absent" %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} +{% if ELASTICSEARCHMERGED.index_clean %} +{% set ap = "present" %} +{% else %} +{% set ap = "absent" %} +{% endif %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +so-elasticsearch-indices-delete: + cron.{{ap}}: + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 + - identifier: so-elasticsearch-indices-delete + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index f4031ee5d..ab12b875e 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,10 +10,6 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} -{% if GLOBALS.role != 'so-heavynode' %} -{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} -{% endif %} include: - ca @@ -21,6 +17,9 @@ include: - elasticsearch.ssl - elasticsearch.config - elasticsearch.sostatus +{%- if GLOBALS.role != 'so-searchode' %} + - elasticsearch.cluster +{%- endif%} so-elasticsearch: docker_container.running: @@ -108,150 +107,19 @@ delete_so-elasticsearch_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-elasticsearch$ -{% if GLOBALS.role != "so-searchnode" %} -escomponenttemplates: - file.recurse: - - name: /opt/so/conf/elasticsearch/templates/component - - source: salt://elasticsearch/templates/component - - user: 930 - - group: 939 - - clean: True - - onchanges_in: - - file: so-elasticsearch-templates-reload - - show_changes: False - -# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory -so_index_template_dir: - file.directory: - - name: /opt/so/conf/elasticsearch/templates/index - - clean: True - {%- if SO_MANAGED_INDICES %} - - require: - {%- for index in SO_MANAGED_INDICES %} - - file: so_index_template_{{index}} - {%- endfor %} - {%- endif %} - -# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) -# These index templates are for the core SO datasets and are always required -{% for index, settings in ES_INDEX_SETTINGS.items() %} -{% if settings.index_template is defined %} -so_index_template_{{index}}: - file.managed: - - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json - - source: salt://elasticsearch/base-template.json.jinja - - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} - - template: jinja - - onchanges_in: - - file: so-elasticsearch-templates-reload -{% endif %} -{% endfor %} - -{% if GLOBALS.role != "so-heavynode" %} -# Auto-generate optional index templates for integration | input | content packages -# These index templates are not used by default (until user adds package to an agent policy). -# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. -{% for index,settings in ALL_ADDON_SETTINGS.items() %} -{% if settings.index_template is defined %} -addon_index_template_{{index}}: - file.managed: - - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json - - source: salt://elasticsearch/base-template.json.jinja - - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} - - template: jinja - - show_changes: False - - onchanges_in: - - file: addon-elasticsearch-templates-reload -{% endif %} -{% endfor %} -{% endif %} - -{% if GLOBALS.role in GLOBALS.manager_roles %} -so-es-cluster-settings: - cmd.run: - - name: /usr/sbin/so-elasticsearch-cluster-settings - - cwd: /opt/so - - template: jinja +wait_for_so-elasticsearch: + http.wait_for_successful_query: + - name: "https://localhost:9200/" + - username: 'so_elastic' + - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + - backend: requests - require: - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja -{% endif %} - -# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) -so-elasticsearch-ilm-policy-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-ilm-policy-load - - cwd: /opt/so - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-ilm-policy-load-script - - onchanges: - - file: so-elasticsearch-ilm-policy-load-script - -so-elasticsearch-templates-reload: - file.absent: - - name: /opt/so/state/estemplates.txt - -addon-elasticsearch-templates-reload: - file.absent: - - name: /opt/so/state/addon_estemplates.txt - -# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script -so-elasticsearch-templates: - cmd.run: -{%- if GLOBALS.role == "so-heavynode" %} - - name: /usr/sbin/so-elasticsearch-templates-load --heavynode -{%- else %} - - name: /usr/sbin/so-elasticsearch-templates-load -{%- endif %} - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -so-elasticsearch-pipelines: - cmd.run: - - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-pipelines-script - -so-elasticsearch-roles-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-roles-load - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %} -{% set ap = "absent" %} -{% endif %} -{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} -{% if ELASTICSEARCHMERGED.index_clean %} -{% set ap = "present" %} -{% else %} -{% set ap = "absent" %} -{% endif %} -{% endif %} -{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} -so-elasticsearch-indices-delete: - cron.{{ap}}: - - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 - - identifier: so-elasticsearch-indices-delete - - user: root - - minute: '*/5' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' -{% endif %} - -{% endif %} {% else %}