diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 278bbde62..d367de287 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,44 +1,43 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} +{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} {% from 'logstash/map.jinja' import LOGSTASH_NODES with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %} {% if LOGSTASH_NODES | length > 1 %} - {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} {% for NODE in LOGSTASH_NODES %} - {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% if grains.id.split('_') | last == 'manager' %} - {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master','data','remote_cluster_client']}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master','data','remote_cluster_client']}) %} {% else %} - {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data_hot', 'remote_cluster_client']}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data_hot', 'remote_cluster_client']}) %} {% endif %} {% endif %} {% elif grains.id.split('_') | last == 'searchnode' %} - {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data_hot', 'ingest']}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['data_hot', 'ingest']}) %} {% if HIGHLANDER %} - {% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %} {% endif %} - {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} {% endif %} {% if HIGHLANDER %} - {% do ESCONFIG.elasticsearch.config.xpack.ml.update({'enabled': true}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %} {% endif %} -{% do ESCONFIG.elasticsearch.config.node.update({'name': GLOBALS.hostname}) %} -{% do ESCONFIG.elasticsearch.config.cluster.update({'name': GLOBALS.hostname}) %} -{% do ESCONFIG.elasticsearch.config.transport.update({'publish_host': GLOBALS.hostname}) %} +{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'name': GLOBALS.hostname}) %} +{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.cluster.update({'name': GLOBALS.hostname}) %} +{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.transport.update({'publish_host': GLOBALS.hostname}) %} -{# merge with the elasticsearch pillar #} -{% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} +{% set ELASTICSEARCHMERGED = salt['pillar.get']('elasticsearch', default=ELASTICSEARCHDEFAULTS.elasticsearch, merge=True) %} {% if salt['pillar.get']('elasticsearch:config:path:repo', False) %} {% for repo in pillar.elasticsearch.config.path.repo %} {# remove elasticsearch.config.path.repo value if the directory doesn't exist on the node #} {% if not salt['file.directory_exists'](repo) %} - {% do ESCONFIG.path.repo.remove(repo) %} + {% do ELASTICSEARCHMERGED.config.path.repo.remove(repo) %} {% endif %} {% endfor %} {% endif %} diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls new file mode 100644 index 000000000..255d09376 --- /dev/null +++ b/salt/elasticsearch/config.sls @@ -0,0 +1,262 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +include: + - ssl + +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} + +vm.max_map_count: + sysctl.present: + - value: 262144 + +# Add ES Group +elasticsearchgroup: + group.present: + - name: elasticsearch + - gid: 930 + +esconfdir: + file.directory: + - name: /opt/so/conf/elasticsearch + - user: 930 + - group: 939 + - makedirs: True + +# Add ES user +elasticsearch: + user.present: + - uid: 930 + - gid: 930 + - home: /opt/so/conf/elasticsearch + - createhome: False + +{% if GLOBALS.is_manager %} +# We have to add the Manager CA to the CA list +cascriptsync: + file.managed: + - name: /usr/sbin/so-catrust + - source: salt://elasticsearch/tools/sbin_jinja/so-catrust + - user: 939 + - group: 939 + - mode: 750 + - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} + +# Run the CA magic +cascriptfun: + cmd.run: + - name: /usr/sbin/so-catrust + - require: + - file: cascriptsync +{% endif %} + +elasticsearch_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://elasticsearch/tools/sbin + - user: 930 + - group: 939 + - file_mode: 755 + - exclude_pat: + - so-catrust + - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state + +elasticsearch_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://elasticsearch/tools/sbin_jinja + - user: 939 + - group: 939 + - file_mode: 755 + - template: jinja + - exclude_pat: + - so-elasticsearch-ilm-policy-load # exclude this because we need to watch it for changes, we sync it in another state + - defaults: + GLOBALS: {{ GLOBALS }} + +so-elasticsearch-ilm-policy-load-script: + file.managed: + - name: /usr/sbin/so-elasticsearch-ilm-policy-load + - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load + - user: 930 + - group: 939 + - mode: 754 + - template: jinja + +so-elasticsearch-pipelines-script: + file.managed: + - name: /usr/sbin/so-elasticsearch-pipelines + - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines + - user: 930 + - group: 939 + - mode: 754 + +# Move our new CA over so Elastic and Logstash can use SSL with the internal CA +catrustdir: + file.directory: + - name: /opt/so/conf/ca + - user: 939 + - group: 939 + - makedirs: True + +cacertz: + file.managed: + - name: /opt/so/conf/ca/cacerts + - source: salt://common/cacerts + - user: 939 + - group: 939 + +capemz: + file.managed: + - name: /opt/so/conf/ca/tls-ca-bundle.pem + - source: salt://common/tls-ca-bundle.pem + - user: 939 + - group: 939 + +esingestdir: + file.directory: + - name: /opt/so/conf/elasticsearch/ingest + - user: 930 + - group: 939 + - makedirs: True + +estemplatedir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/index + - user: 930 + - group: 939 + - makedirs: True + +esrolesdir: + file.directory: + - name: /opt/so/conf/elasticsearch/roles + - user: 930 + - group: 939 + - makedirs: True + +eslibdir: + file.absent: + - name: /opt/so/conf/elasticsearch/lib + +esingestdynamicconf: + file.recurse: + - name: /opt/so/conf/elasticsearch/ingest + - source: salt://elasticsearch/files/ingest-dynamic + - user: 930 + - group: 939 + - template: jinja + +esingestconf: + file.recurse: + - name: /opt/so/conf/elasticsearch/ingest + - source: salt://elasticsearch/files/ingest + - user: 930 + - group: 939 + +eslog4jfile: + file.managed: + - name: /opt/so/conf/elasticsearch/log4j2.properties + - source: salt://elasticsearch/files/log4j2.properties + - user: 930 + - group: 939 + - template: jinja + +esyml: + file.managed: + - name: /opt/so/conf/elasticsearch/elasticsearch.yml + - source: salt://elasticsearch/files/elasticsearch.yaml.jinja + - user: 930 + - group: 939 + - defaults: + ESCONFIG: {{ ELASTICSEARCHMERGED.config }} + - template: jinja + +esroles: + file.recurse: + - source: salt://elasticsearch/roles/ + - name: /opt/so/conf/elasticsearch/roles/ + - clean: True + - template: jinja + - user: 930 + - group: 939 + +nsmesdir: + file.directory: + - name: /nsm/elasticsearch + - user: 930 + - group: 939 + - makedirs: True + +eslogdir: + file.directory: + - name: /opt/so/log/elasticsearch + - user: 930 + - group: 939 + - makedirs: True + +es_repo_dir: + file.directory: + - name: /nsm/elasticsearch/repo/ + - user: 930 + - group: 930 + - require: + - file: nsmesdir + +so-pipelines-reload: + file.absent: + - name: /opt/so/state/espipelines.txt + - onchanges: + - file: esingestconf + - file: esingestdynamicconf + - file: esyml + - file: so-elasticsearch-pipelines-script + +auth_users: + file.managed: + - name: /opt/so/conf/elasticsearch/users.tmp + - source: salt://elasticsearch/files/users + - user: 930 + - group: 930 + - mode: 600 + - show_changes: False + +auth_users_roles: + file.managed: + - name: /opt/so/conf/elasticsearch/users_roles.tmp + - source: salt://elasticsearch/files/users_roles + - user: 930 + - group: 930 + - mode: 600 + - show_changes: False + +auth_users_inode: + require: + - file: auth_users + cmd.run: + - name: cat /opt/so/conf/elasticsearch/users.tmp > /opt/so/conf/elasticsearch/users && chown 930:939 /opt/so/conf/elasticsearch/users && chmod 660 /opt/so/conf/elasticsearch/users + - onchanges: + - file: /opt/so/conf/elasticsearch/users.tmp + +auth_users_roles_inode: + require: + - file: auth_users_roles + cmd.run: + - name: cat /opt/so/conf/elasticsearch/users_roles.tmp > /opt/so/conf/elasticsearch/users_roles && chown 930:939 /opt/so/conf/elasticsearch/users_roles && chmod 660 /opt/so/conf/elasticsearch/users_roles + - onchanges: + - file: /opt/so/conf/elasticsearch/users_roles.tmp + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 37f55ddc7..10cc347d1 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,4 +1,5 @@ elasticsearch: + enabled: False retention: retention_pct: 50 config: diff --git a/salt/elasticsearch/disabled.sls b/salt/elasticsearch/disabled.sls new file mode 100644 index 000000000..210ad59a4 --- /dev/null +++ b/salt/elasticsearch/disabled.sls @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +include: + - elasticsearch.sostatus + +so-elasticsearch: + docker_container.absent: + - force: True + +so-elasticsearch_so-status.disabled: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-elasticsearch$ + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls new file mode 100644 index 000000000..0fc77f0ee --- /dev/null +++ b/salt/elasticsearch/enabled.sls @@ -0,0 +1,182 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'logstash/map.jinja' import LOGSTASH_NODES %} +{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} + +include: + - elasticsearch.config + - elasticsearch.sostatus + +so-elasticsearch: + docker_container.running: + - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }} + - hostname: elasticsearch + - name: so-elasticsearch + - user: elasticsearch + - networks: + - sobridge: + - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} + - extra_hosts: {{ LOGSTASH_NODES }} + - environment: + {% if LOGSTASH_NODES | length == 1 %} + - discovery.type=single-node + {% endif %} + - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true + ulimits: + - memlock=-1:-1 + - nofile=65536:65536 + - nproc=4096 + - port_bindings: + {% for BINDING in DOCKER.containers['so-elasticsearch'].port_bindings %} + - {{ BINDING }} + {% endfor %} + - binds: + - /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro + - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw + - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw + - /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro + {% if GLOBALS.is_manager %} + - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro + {% else %} + - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro + {% endif %} + - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro + - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro + - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro + - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro + - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro + {% if ELASTICSEARCHMERGED.config.path.get('repo', False) %} + {% for repo in ELASTICSEARCHMERGED.config.path.repo %} + - {{ repo }}:{{ repo }}:rw + {% endfor %} + {% endif %} + - watch: + - file: cacertz + - file: esyml + - require: + - file: esyml + - file: eslog4jfile + - file: nsmesdir + - file: eslogdir + - file: cacertz + - x509: /etc/pki/elasticsearch.crt + - x509: /etc/pki/elasticsearch.key + - file: elasticp12perms + {% if GLOBALS.is_manager %} + - x509: pki_public_ca_crt + {% else %} + - x509: trusttheca + {% endif %} + - cmd: auth_users_roles_inode + - cmd: auth_users_inode + +delete_so-elasticsearch_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-elasticsearch$ + +{% if GLOBALS.role != "so-searchnode" %} +escomponenttemplates: + file.recurse: + - name: /opt/so/conf/elasticsearch/templates/component + - source: salt://elasticsearch/templates/component + - user: 930 + - group: 939 + - onchanges_in: + - cmd: so-elasticsearch-templates + +# Auto-generate templates from defaults file +{% for index, settings in ES_INDEX_SETTINGS.items() %} + {% if settings.index_template is defined %} +es_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja + - onchanges_in: + - cmd: so-elasticsearch-templates +{% endif %} +{% endfor %} + +{% if TEMPLATES %} +# Sync custom templates to /opt/so/conf/elasticsearch/templates +{% for TEMPLATE in TEMPLATES %} +es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: + file.managed: + - source: salt://elasticsearch/templates/index/{{TEMPLATE}} +{% if 'jinja' in TEMPLATE.split('.')[-1] %} + - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} + - template: jinja +{% else %} + - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}} +{% endif %} + - user: 930 + - group: 939 + - onchanges_in: + - cmd: so-elasticsearch-templates +{% endfor %} +{% endif %} + +so-es-cluster-settings: + cmd.run: + - name: /usr/sbin/so-elasticsearch-cluster-settings + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +so-elasticsearch-ilm-policy-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-ilm-policy-load + - cwd: /opt/so + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-ilm-policy-load-script + - onchanges: + - file: so-elasticsearch-ilm-policy-load-script + +so-elasticsearch-templates: + cmd.run: + - name: /usr/sbin/so-elasticsearch-templates-load + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +so-elasticsearch-pipelines: + cmd.run: + - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-pipelines-script + +so-elasticsearch-roles-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-roles-load + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 5e2ffae9d..301c836c2 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -3,425 +3,11 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} include: - - ssl - -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} -{% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %} -{% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %} -{% from 'logstash/map.jinja' import LOGSTASH_NODES %} - -vm.max_map_count: - sysctl.present: - - value: 262144 - -# Add ES Group -elasticsearchgroup: - group.present: - - name: elasticsearch - - gid: 930 - -esconfdir: - file.directory: - - name: /opt/so/conf/elasticsearch - - user: 930 - - group: 939 - - makedirs: True - -# Add ES user -elasticsearch: - user.present: - - uid: 930 - - gid: 930 - - home: /opt/so/conf/elasticsearch - - createhome: False - -{% if GLOBALS.is_manager %} -# We have to add the Manager CA to the CA list -cascriptsync: - file.managed: - - name: /usr/sbin/so-catrust - - source: salt://elasticsearch/tools/sbin_jinja/so-catrust - - user: 939 - - group: 939 - - mode: 750 - - template: jinja - - defaults: - GLOBALS: {{ GLOBALS }} - -# Run the CA magic -cascriptfun: - cmd.run: - - name: /usr/sbin/so-catrust - - require: - - file: cascriptsync -{% endif %} - -elasticsearch_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://elasticsearch/tools/sbin - - user: 930 - - group: 939 - - file_mode: 755 - - exclude_pat: - - so-catrust - - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state - -elasticsearch_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://elasticsearch/tools/sbin_jinja - - user: 939 - - group: 939 - - file_mode: 755 - - template: jinja - - exclude_pat: - - so-elasticsearch-ilm-policy-load # exclude this because we need to watch it for changes, we sync it in another state - - defaults: - GLOBALS: {{ GLOBALS }} - -so-elasticsearch-ilm-policy-load-script: - file.managed: - - name: /usr/sbin/so-elasticsearch-ilm-policy-load - - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load - - user: 930 - - group: 939 - - mode: 754 - - template: jinja - -so-elasticsearch-pipelines-script: - file.managed: - - name: /usr/sbin/so-elasticsearch-pipelines - - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines - - user: 930 - - group: 939 - - mode: 754 - -# Move our new CA over so Elastic and Logstash can use SSL with the internal CA -catrustdir: - file.directory: - - name: /opt/so/conf/ca - - user: 939 - - group: 939 - - makedirs: True - -cacertz: - file.managed: - - name: /opt/so/conf/ca/cacerts - - source: salt://common/cacerts - - user: 939 - - group: 939 - -capemz: - file.managed: - - name: /opt/so/conf/ca/tls-ca-bundle.pem - - source: salt://common/tls-ca-bundle.pem - - user: 939 - - group: 939 - -esingestdir: - file.directory: - - name: /opt/so/conf/elasticsearch/ingest - - user: 930 - - group: 939 - - makedirs: True - -estemplatedir: - file.directory: - - name: /opt/so/conf/elasticsearch/templates/index - - user: 930 - - group: 939 - - makedirs: True - -esrolesdir: - file.directory: - - name: /opt/so/conf/elasticsearch/roles - - user: 930 - - group: 939 - - makedirs: True - -eslibdir: - file.absent: - - name: /opt/so/conf/elasticsearch/lib - -esingestdynamicconf: - file.recurse: - - name: /opt/so/conf/elasticsearch/ingest - - source: salt://elasticsearch/files/ingest-dynamic - - user: 930 - - group: 939 - - template: jinja - -esingestconf: - file.recurse: - - name: /opt/so/conf/elasticsearch/ingest - - source: salt://elasticsearch/files/ingest - - user: 930 - - group: 939 - -eslog4jfile: - file.managed: - - name: /opt/so/conf/elasticsearch/log4j2.properties - - source: salt://elasticsearch/files/log4j2.properties - - user: 930 - - group: 939 - - template: jinja - -esyml: - file.managed: - - name: /opt/so/conf/elasticsearch/elasticsearch.yml - - source: salt://elasticsearch/files/elasticsearch.yaml.jinja - - user: 930 - - group: 939 - - defaults: - ESCONFIG: {{ ESCONFIG }} - - template: jinja - -{% if GLOBALS.role != "so-searchnode" %} -escomponenttemplates: - file.recurse: - - name: /opt/so/conf/elasticsearch/templates/component - - source: salt://elasticsearch/templates/component - - user: 930 - - group: 939 - - onchanges_in: - - cmd: so-elasticsearch-templates - -# Auto-generate templates from defaults file -{% for index, settings in ES_INDEX_SETTINGS.items() %} - {% if settings.index_template is defined %} -es_index_template_{{index}}: - file.managed: - - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json - - source: salt://elasticsearch/base-template.json.jinja - - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} - - template: jinja - - onchanges_in: - - cmd: so-elasticsearch-templates - {% endif %} -{% endfor %} - -{% if TEMPLATES %} -# Sync custom templates to /opt/so/conf/elasticsearch/templates -{% for TEMPLATE in TEMPLATES %} -es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: - file.managed: - - source: salt://elasticsearch/templates/index/{{TEMPLATE}} - {% if 'jinja' in TEMPLATE.split('.')[-1] %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} - - template: jinja - {% else %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}} - {% endif %} - - user: 930 - - group: 939 - - onchanges_in: - - cmd: so-elasticsearch-templates -{% endfor %} -{% endif %} -{% endif %} - -esroles: - file.recurse: - - source: salt://elasticsearch/roles/ - - name: /opt/so/conf/elasticsearch/roles/ - - clean: True - - template: jinja - - user: 930 - - group: 939 - -nsmesdir: - file.directory: - - name: /nsm/elasticsearch - - user: 930 - - group: 939 - - makedirs: True - -eslogdir: - file.directory: - - name: /opt/so/log/elasticsearch - - user: 930 - - group: 939 - - makedirs: True - -es_repo_dir: - file.directory: - - name: /nsm/elasticsearch/repo/ - - user: 930 - - group: 930 - - require: - - file: nsmesdir - -so-pipelines-reload: - file.absent: - - name: /opt/so/state/espipelines.txt - - onchanges: - - file: esingestconf - - file: esingestdynamicconf - - file: esyml - - file: so-elasticsearch-pipelines-script - -auth_users: - file.managed: - - name: /opt/so/conf/elasticsearch/users.tmp - - source: salt://elasticsearch/files/users - - user: 930 - - group: 930 - - mode: 600 - - show_changes: False - -auth_users_roles: - file.managed: - - name: /opt/so/conf/elasticsearch/users_roles.tmp - - source: salt://elasticsearch/files/users_roles - - user: 930 - - group: 930 - - mode: 600 - - show_changes: False - -auth_users_inode: - require: - - file: auth_users - cmd.run: - - name: cat /opt/so/conf/elasticsearch/users.tmp > /opt/so/conf/elasticsearch/users && chown 930:939 /opt/so/conf/elasticsearch/users && chmod 660 /opt/so/conf/elasticsearch/users - - onchanges: - - file: /opt/so/conf/elasticsearch/users.tmp - -auth_users_roles_inode: - require: - - file: auth_users_roles - cmd.run: - - name: cat /opt/so/conf/elasticsearch/users_roles.tmp > /opt/so/conf/elasticsearch/users_roles && chown 930:939 /opt/so/conf/elasticsearch/users_roles && chmod 660 /opt/so/conf/elasticsearch/users_roles - - onchanges: - - file: /opt/so/conf/elasticsearch/users_roles.tmp - -so-elasticsearch: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }} - - hostname: elasticsearch - - name: so-elasticsearch - - user: elasticsearch - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - - extra_hosts: {{ LOGSTASH_NODES }} - - environment: - {% if LOGSTASH_NODES | length == 1 %} - - discovery.type=single-node - {% endif %} - - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true - ulimits: - - memlock=-1:-1 - - nofile=65536:65536 - - nproc=4096 - - port_bindings: - {% for BINDING in DOCKER.containers['so-elasticsearch'].port_bindings %} - - {{ BINDING }} - {% endfor %} - - binds: - - /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro - - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro - - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw - - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - - /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro - {% if GLOBALS.is_manager %} - - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro - {% else %} - - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro - {% endif %} - - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro - - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro - - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro - {% if ESCONFIG.path.get('repo', False) %} - {% for repo in ESCONFIG.path.repo %} - - {{ repo }}:{{ repo }}:rw - {% endfor %} - {% endif %} - - watch: - - file: cacertz - - file: esyml - - require: - - file: esyml - - file: eslog4jfile - - file: nsmesdir - - file: eslogdir - - file: cacertz - - x509: /etc/pki/elasticsearch.crt - - x509: /etc/pki/elasticsearch.key - - file: elasticp12perms - {% if GLOBALS.is_manager %} - - x509: pki_public_ca_crt - {% else %} - - x509: trusttheca - {% endif %} - - cmd: auth_users_roles_inode - - cmd: auth_users_inode - -append_so-elasticsearch_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-elasticsearch - -{% if GLOBALS.role != "so-searchnode" %} - -so-es-cluster-settings: - cmd.run: - - name: /usr/sbin/so-elasticsearch-cluster-settings - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -so-elasticsearch-ilm-policy-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-ilm-policy-load - - cwd: /opt/so - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-ilm-policy-load-script - - onchanges: - - file: so-elasticsearch-ilm-policy-load-script - -so-elasticsearch-templates: - cmd.run: - - name: /usr/sbin/so-elasticsearch-templates-load - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -so-elasticsearch-pipelines: - cmd.run: - - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-pipelines-script - -so-elasticsearch-roles-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-roles-load - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja -{% endif %} +{% if ELASTICSEARCHMERGED.enabled %} + - elasticsearch.enabled {% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} {# if 'elasticsearch' in top_states #} + - elasticsearch.disabled +{% endif %} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index ded8e5be6..da22268f6 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -1,4 +1,7 @@ elasticsearch: + enabled: + description: You can enable or disable Elasticsearch. + helpLink: elasticsearch.html esheap: description: Specify the memory heap size in (m)egabytes for Elasticsearch. helpLink: elasticsearch.html diff --git a/salt/elasticsearch/sostatus.sls b/salt/elasticsearch/sostatus.sls new file mode 100644 index 000000000..2967a39db --- /dev/null +++ b/salt/elasticsearch/sostatus.sls @@ -0,0 +1,21 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +append_so-elasticsearch_so-status.conf: + file.append: + - name: /opt/so/conf/so-status/so-status.conf + - text: so-elasticsearch + - unless: grep -q so-elasticsearch /opt/so/conf/so-status/so-status.conf + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 9c90cc28f..49d86d187 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,5 +1,5 @@ -{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} +{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} +{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} {% for index, settings in ES_INDEX_SETTINGS.items() %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index afeddfa01..afb8bdc67 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -6,15 +6,15 @@ . /usr/sbin/so-common -{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} +{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} - {%- if settings.policy is defined %} +{%- if settings.policy is defined %} echo echo "Setting up {{ index }}-logs policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo - {%- endif %} +{%- endif %} {%- endfor %} echo diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index b94bfac59..0195cdb5c 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -125,9 +125,10 @@ function create_minion_files() { } # Add Elastic settings to the minion file -function add_elastic_to_minion() { +function add_elasticsearch_to_minion() { printf '%s\n'\ "elasticsearch:"\ + " enabled: True"\ " esheap: '$ES_HEAP_SIZE'"\ " " >> $PILLARFILE } @@ -279,14 +280,14 @@ function apply_ES_state() { salt-call state.apply elasticsearch concurrent=True } function createEVAL() { - add_elastic_to_minion + add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion add_elastalert_to_minion } function createSTANDALONE() { - add_elastic_to_minion + add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion add_playbook_to_minion @@ -294,14 +295,14 @@ function createSTANDALONE() { } function createMANAGER() { - add_elastic_to_minion + add_elasticsearch_to_minion add_logstash_to_minion add_playbook_to_minion add_elastalert_to_minion } function createMANAGERSEARCH() { - add_elastic_to_minion + add_elasticsearch_to_minion add_logstash_to_minion add_playbook_to_minion add_elastalert_to_minion @@ -320,13 +321,13 @@ function createIDH() { } function createIMPORT() { - add_elastic_to_minion + add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion } function createHEAVYNODE() { - add_elastic_to_minion + add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion } @@ -336,7 +337,7 @@ function createSENSOR() { } function createSEARCHNODE() { - add_elastic_to_minion + add_elasticsearch_to_minion add_logstash_to_minion updateMine apply_ES_state diff --git a/salt/pcap/sostatus.sls b/salt/pcap/sostatus.sls index d7380e804..41ff1a183 100644 --- a/salt/pcap/sostatus.sls +++ b/salt/pcap/sostatus.sls @@ -1,3 +1,8 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %}