Merge branch '2.4/dev' into foxtrot

salt/elasticsearch/config.map.jinja

@@ -1,23 +1,37 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
 
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
-{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #}
-{% set ES_LOGSTASH_NODES = [] %}
-{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
+{% set ELASTICSEARCH_SEED_HOSTS = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 {% for node_type, node_details in node_data.items() | sort %}
-{%   if node_type not in ['heavynode', 'fleet'] %}
+{%   if node_type != 'heavynode' %}
 {%     for hostname in node_data[node_type].keys() %}
-{%       do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%       do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %}
 {%     endfor %}
 {%   endif %}
 {% endfor %}
 
+{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #}
+{% set ELASTICSEARCH_NODES = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% for node_type, node_details in node_data.items() %}
+{%   for hostname in node_data[node_type].keys() %}
+{%     do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%   endfor %}
+{% endfor %}
+
 {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
-    {% if ES_LOGSTASH_NODES | length > 1 %}
+    {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
       {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
-      {% for NODE in ES_LOGSTASH_NODES %}
+      {% for NODE in ELASTICSEARCH_SEED_HOSTS %}
         {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
       {% endfor %}
     {% endif %}

salt/elasticsearch/enabled.sls

@@ -7,8 +7,8 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
-{%   from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -27,7 +27,7 @@ so-elasticsearch:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
     - extra_hosts:
-    {% for node in LOGSTASH_NODES %}
+    {% for node in ELASTICSEARCH_NODES %}
     {%   for hostname, ip in node.items() %}
       - {{hostname}}:{{ip}}
     {%   endfor %}
@@ -38,7 +38,7 @@ so-elasticsearch:
       {% endfor %}
     {% endif %}
     - environment:
-      {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
+      {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true

salt/elasticsearch/soc_elasticsearch.yaml

@@ -530,58 +530,6 @@ elasticsearch:
     so-strelka: *indexSettings
     so-syslog: *indexSettings
     so-zeek: *indexSettings
-    so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
-      index_sorting:
-        description: Sorts the index by event time, at the cost of additional processing resource consumption.
-        advanced: True
-        readonly: True
-        helpLink: elasticsearch.html
-      index_template:
-        ignore_missing_component_templates:
-          description: Ignore component templates if they aren't in Elasticsearch.
-          advanced: True
-          readonly: True
-          helpLink: elasticsearch.html
-        index_patterns:
-          description: Patterns for matching multiple indices or tables.
-          advanced: True
-          readonly: True
-          helpLink: elasticsearch.html
-        template:
-          settings:
-            index:
-              mode:
-                description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage.
-                advanced: True
-                readonly: True
-                helpLink: elasticsearch.html
-              number_of_replicas:
-                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
-                advanced: True
-                readonly: True
-                helpLink: elasticsearch.html
-        composed_of:
-          description: The index template is composed of these component templates.
-          advanced: True
-          readonly: True
-          helpLink: elasticsearch.html
-        priority:
-          description: The priority of the index template.
-          advanced: True
-          readonly: True
-          helpLink: elasticsearch.html
-        data_stream:
-          hidden:
-            description: Hide the data stream.
-            advanced: True
-            readonly: True
-            helpLink: elasticsearch.html
-          allow_custom_routing:
-            description: Allow custom routing for the data stream.
-            advanced: True
-            readonly: True
-            helpLink: elasticsearch.html
-    so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings
   so_roles:
     so-manager: &soroleSettings
       config:

salt/elasticsearch/template.map.jinja

@@ -1,3 +1,8 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
 
@@ -17,10 +22,26 @@
 {% set ES_INDEX_SETTINGS = {} %}
 {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
-{#   if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #}
-{#   this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%   if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%     do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+
+{#   prevent this action from being performed on custom defined indices. #}
+{#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+
+{#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
+{#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     endif %}
+
+{#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%         endif %}
+{%       endfor %}
+{%     endif %}
+
 {%   endif %}
 
 {%   if settings.index_template is defined %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {%- from 'vars/globals.map.jinja' import GLOBALS %}
-{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 
 . /usr/sbin/so-common
 

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -134,7 +134,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
       MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
       else
@@ -153,7 +153,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
   cd - >/dev/null
 
   if [[ $load_failures -eq 0 ]]; then
-    echo "All templates loaded successfully"
+    echo "All template loaded successfully"
     touch $STATE_FILE_SUCCESS
   else
     echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"