WIP: support all es fleet integrations

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+
+. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
+
+# Check that /opt/so/state/estemplates.txt exists to signal that Elasticsearch
+#  has completed its first run of core-only integrations/indices/components/ilm
+STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
+INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
+BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
+BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
+PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+
+SKIP_SUBSCRIPTION=true
+PENDING_UPDATE=false
+
+version_conversion(){
+    version=$1
+    echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
+}
+
+compare_versions() {
+    version1=$1
+    version2=$2
+
+    # Convert versions to numbers
+    num1=$(version_conversion "$version1")
+    num2=$(version_conversion "$version2")
+
+    # Compare using bc
+    if (( $(echo "$num1 < $num2" | bc -l) )); then
+        echo "less"
+    elif (( $(echo "$num1 > $num2" | bc -l) )); then
+        echo "greater"
+    else
+        echo "equal"
+    fi
+}
+
+if [[ -f $STATE_FILE_SUCCESS  ]]; then
+    if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then
+        # Package_list contains all NON-beta integrations.
+        latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list)
+        echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST
+        rm -f $INSTALLED_PACKAGE_LIST
+        echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
+
+        cat "$INSTALLED_PACKAGE_LIST" | jq -c '.packages[]' | while read -r package; do
+            # get package details
+            package_name=$(echo "$package" | jq -r '.name')
+            latest_version=$(echo "$package" | jq -r '.latest_version')
+            installed_version=$(echo "$package" | jq -r '.installed_version')
+            subscription=$(echo "$package" | jq -r '.subscription')
+            bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
+
+            if [ $SKIP_SUBSCRIPTION ] && [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
+                # pass over integrations that require non-basic elastic license
+                continue
+            else
+                if [ -n "$installed_version" ]; then
+                    results=$(compare_versions "$latest_version" "$installed_version")
+                    if [ $results == "greater" ]; then
+                        echo "$package_name is not up to date... Adding to next update."
+                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                        PENDING_UPDATE=true
+                    fi
+                else
+                    echo "$package_name is not installed... Adding to next update."
+                    jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                    PENDING_UPDATE=true
+                fi
+            fi
+        done
+
+        if [ $PENDING_UPDATE ]; then
+            # Run bulk install of packages
+            # elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST
+
+            # Write out file for generating index/component/ilm templates
+            latest_installed_package_list=$(elastic_fleet_installed_packages)
+            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+
+        else
+            echo "Elastic integrations don't appear to need installation/updating..."
+            exit 0
+        fi
+
+    else
+        # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.
+        echo "Elastic Fleet does not appear to be responding... Exiting... "
+        exit 0
+    fi
+else
+    # This message will appear when an update to core integration is made and this script is run at the same time as
+    #  elasticsearch.enabled -> detects change to core index settings -> deletes estemplates.txt
+    echo "Elasticsearch may not be fully configured yet or is currently updating core index settings."
+    exit 0
+fi

salt/elasticfleet/defaults.yaml

@@ -10,6 +10,7 @@ elasticfleet:
       grid_enrollment: ''
     defend_filters:
       enable_auto_configuration: False
+    subscription_integrations: False
   logging:
     zeek:
       excluded:

salt/elasticfleet/integration-defaults.map.jinja

@@ -0,0 +1,78 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{% import_yaml 'elasticfleet/integration-defaults.yaml' as INTEGRATIONDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_INTEGRATION_DEFAULTS = {} %}
+
+{% for pkg in ADDON_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core integrations #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each integration #}
+{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
+{%       for pattern in pkg.es_index_patterns %}
+{%         set integration_key = "so-logs-" ~ pkg.name ~ "_x_" ~ pattern.title %}
+{%         set integration_defaults = {
+              "index_sorting": false,
+              "index_template": {
+                  "composed_of": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@package", "logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                  "data_stream": {
+                      "hidden": false,
+                      "allow_custom_routing": false
+                  },
+                  "ignore_missing_component_templates": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom"],
+                  "index_patterns": [pattern.name],
+                  "priority": 501,
+                  "template": {
+                      "settings": {
+                          "index": {
+                              "lifecycle": {"name": "so-logs-" ~ pkg.name ~ "." ~ pattern.title ~ "-logs"},
+                              "number_of_replicas": 0
+                          }
+                      }
+                  }
+              },
+              "policy": {
+                  "phases": {
+                      "cold": {
+                          "actions": {
+                              "set_priority": {"priority": 0}
+                          },
+                          "min_age": "60d"
+                      },
+                      "delete": {
+                          "actions": {
+                              "delete": {}
+                          },
+                          "min_age": "365d"
+                      },
+                      "hot": {
+                          "actions": {
+                              "rollover": {
+                                  "max_age": "30d",
+                                  "max_primary_shard_size": "50gb"
+                              },
+                              "set_priority": {"priority": 100}
+                          },
+                          "min_age": "0ms"
+                      },
+                      "warm": {
+                          "actions": {
+                              "set_priority": {"priority": 50}
+                          },
+                          "min_age": "30d"
+                      }
+                  }
+              }
+          } %}
+{%         do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%       endfor %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/integration-defaults.yaml

@@ -0,0 +1,46 @@
+so-logs-INTPLACEHOLDER_x_COMPLACEHOLDER:
+  index_sorting: False
+  index_template:
+    composed_of:
+      - "logs-INTPLACEHOLDER.COMPLACEHOLDER@package"
+      - "logs-INTPLACEHOLDER.COMPLACEHOLDER@custom"
+      - "so-fleet_globals-1"
+      - "so-fleet_agent_id_verification-1"
+    data_stream:
+      hidden: false
+      allow_custom_routing: false
+    ignore_missing_COMPLACEHOLDER_templates:
+      - "logs-INTPLACEHOLDER.COMPLACEHOLDER@custom"
+    index_patterns:
+      - "logs-INTPLACEHOLDER.COMPLACEHOLDER-*"
+    priority: 501
+    template:
+      settings:
+        index:
+          lifecycle:
+            name: "so-logs-INTPLACEHOLDER.COMPLACEHOLDER-logs"
+          number_of_replicas: 0
+  policy:
+    phases:
+      cold:
+        actions:
+          set_priority:
+            priority: 0
+        min_age: "60d"
+      delete:
+        actions:
+          delete: {}
+        min_age: "365d"
+      hot:
+        actions:
+          rollover:
+            max_age: "30d"
+            max_primary_shard_size: "50gb"
+          set_priority:
+            priority: 100
+        min_age: "0ms"
+      warm:
+        actions:
+          set_priority:
+            priority: 50
+        min_age: "30d"

salt/elasticfleet/soc_elasticfleet.yaml

@@ -40,6 +40,11 @@ elasticfleet:
         global: True
         helpLink: elastic-fleet.html
         advanced: True
+    subscription_integrations:
+      description: Enable the installation of integrations that require an Elastic license.
+      global: True
+      forcedType: bool
+      helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -97,11 +97,20 @@ elastic_fleet_package_install() {
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
 }
 
+elastic_fleet_bulk_package_install() {
+    BULK_PKG_LIST=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
+}
+
 elastic_fleet_package_is_installed() {
     PACKAGE=$1
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
 }
 
+elastic_fleet_installed_packages() {
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=300"
+}
+
 elastic_fleet_agent_policy_ids() {
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
     if [ $? -ne 0 ]; then

salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list

@@ -10,6 +10,6 @@
 SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
 
 # List configured package policies
-curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages" -H 'kbn-xsrf: true' | jq
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages?prerelease=true" -H 'kbn-xsrf: true' | jq
 
 echo

salt/elasticsearch/integration-templates.map.jinja

@@ -0,0 +1,110 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{% set packages = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set INTEGRATION_INDEX_SETTINGS = {} %}
+
+
+{% set default_settings = {
+    'index_sorting': false,
+    'index_template': {
+        'data_stream': {
+          'allow_custom_routing': false,
+          'hidden': false
+        },
+        'priority': 501,
+        'template': {
+            'settings': {
+                'index': {
+                    'number_of_replicas': 0
+                }
+            }
+        }
+    },
+    'policy': {
+        'phases': {
+          'cold': {
+            'actions': {
+              'set_priority': {
+                'priority': 0
+              }
+            },
+            'min_age': '60d'
+          },
+          'delete': {
+            'actions': {
+              'delete': {}
+            },
+            'min_age': '365d'
+          },
+          'hot': {
+            'actions': {
+              'rollover':{
+                'max_age': '30d',
+                'max_primary_shard_size': '50gb'
+              },
+              'set_priority': {
+                'priority': 100
+              }
+            },
+            'min_age': '0ms'
+          },
+          'warm': {
+            'actions': {
+              'set_priority': {
+                'priority': 50
+              }
+            },
+            'min_age': '30d'
+          }
+        }
+    }
+} %}
+
+{# Create template for each package component from elasticfleet/defaults.yaml #}
+{% for package in packages %}
+    {% for pkg_name, components in package.items() %}
+        {% if components is not none %}
+            {% for component in components %}
+                {% set component_dot = component.replace('_x_', '.') %}
+                {% set template_name = 'so-logs-' ~ component %}
+
+                {% set template = {
+                    'index_sorting': default_settings.index_sorting,
+                    'index_template': {
+                        'composed_of': [
+                            'logs-' ~ component_dot ~ '@package',
+                            'logs-' ~ component_dot ~ '@custom',
+                            'so-fleet-_globals-1',
+                            'so-fleet_agent_id_verification-1'
+                        ],
+                        'data_stream': default_settings.index_template.data_stream,
+                        'ignore_missing_component_templates': [
+                            'logs-' ~ component_dot ~  '@custom'
+                        ],
+                        'index_patterns': [
+                            'logs-' ~ component_dot ~ '-*'
+                        ],
+                        'priority': default_settings.index_template.priority,
+                        'template': {
+                            'settings': {
+                                'index': {
+                                    'lifecycle': {
+                                        'name': 'so-logs-' ~ component_dot ~ '-logs'
+                                    },
+                                    'number_of_replicas': default_settings.index_template.template.settings.index.number_of_replicas
+                                }
+                            }
+                        }
+                    },
+                    'policy': default_settings.policy
+                } %}
+
+                {% do INTEGRATION_INDEX_SETTINGS.update({template_name: template}) %}
+            {% endfor %}
+        {% endif %}
+    {% endfor %}
+{% endfor %}

salt/elasticsearch/template.map.jinja

@@ -14,6 +14,15 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
+{# start generation of integration default index_settings #}
+{% if salt['file.file_exists']('/opt/so/state/estemplates.txt') %}
+{%   from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
+{%   for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
+{%     do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
+{%   endfor %}
+{% endif %}
+{# end generation of integration default index_settings #}
+
 {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %}
 {% for index in ES_INDEX_SETTINGS_ORIG.keys() %}
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}