diff --git a/salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load b/salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load new file mode 100644 index 000000000..d94b006ad --- /dev/null +++ b/salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load @@ -0,0 +1,102 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. + +. /usr/sbin/so-common +. /usr/sbin/so-elastic-fleet-common + +# Check that /opt/so/state/estemplates.txt exists to signal that Elasticsearch +# has completed its first run of core-only integrations/indices/components/ilm +STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt +INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json +BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json +BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json +PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json + +SKIP_SUBSCRIPTION=true +PENDING_UPDATE=false + +version_conversion(){ + version=$1 + echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }' +} + +compare_versions() { + version1=$1 + version2=$2 + + # Convert versions to numbers + num1=$(version_conversion "$version1") + num2=$(version_conversion "$version2") + + # Compare using bc + if (( $(echo "$num1 < $num2" | bc -l) )); then + echo "less" + elif (( $(echo "$num1 > $num2" | bc -l) )); then + echo "greater" + else + echo "equal" + fi +} + +if [[ -f $STATE_FILE_SUCCESS ]]; then + if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then + # Package_list contains all NON-beta integrations. + latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list) + echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST + rm -f $INSTALLED_PACKAGE_LIST + echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST + + cat "$INSTALLED_PACKAGE_LIST" | jq -c '.packages[]' | while read -r package; do + # get package details + package_name=$(echo "$package" | jq -r '.name') + latest_version=$(echo "$package" | jq -r '.latest_version') + installed_version=$(echo "$package" | jq -r '.installed_version') + subscription=$(echo "$package" | jq -r '.subscription') + bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' ) + + if [ $SKIP_SUBSCRIPTION ] && [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then + # pass over integrations that require non-basic elastic license + continue + else + if [ -n "$installed_version" ]; then + results=$(compare_versions "$latest_version" "$installed_version") + if [ $results == "greater" ]; then + echo "$package_name is not up to date... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + fi + else + echo "$package_name is not installed... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + fi + fi + done + + if [ $PENDING_UPDATE ]; then + # Run bulk install of packages + # elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST + + # Write out file for generating index/component/ilm templates + latest_installed_package_list=$(elastic_fleet_installed_packages) + echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS + + else + echo "Elastic integrations don't appear to need installation/updating..." + exit 0 + fi + + else + # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run. + echo "Elastic Fleet does not appear to be responding... Exiting... " + exit 0 + fi +else + # This message will appear when an update to core integration is made and this script is run at the same time as + # elasticsearch.enabled -> detects change to core index settings -> deletes estemplates.txt + echo "Elasticsearch may not be fully configured yet or is currently updating core index settings." + exit 0 +fi diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 41c50a96d..a0f509136 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -10,6 +10,7 @@ elasticfleet: grid_enrollment: '' defend_filters: enable_auto_configuration: False + subscription_integrations: False logging: zeek: excluded: diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja new file mode 100644 index 000000000..9977856c4 --- /dev/null +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -0,0 +1,78 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{% import_yaml 'elasticfleet/integration-defaults.yaml' as INTEGRATIONDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_INTEGRATION_DEFAULTS = {} %} + +{% for pkg in ADDON_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core integrations #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each integration #} +{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %} +{% for pattern in pkg.es_index_patterns %} +{% set integration_key = "so-logs-" ~ pkg.name ~ "_x_" ~ pattern.title %} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@package", "logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "hidden": false, + "allow_custom_routing": false + }, + "ignore_missing_component_templates": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom"], + "index_patterns": [pattern.name], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-logs-" ~ pkg.name ~ "." ~ pattern.title ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} +{% do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% endfor %} +{% endif %} +{% endif %} +{% endfor %} \ No newline at end of file diff --git a/salt/elasticfleet/integration-defaults.yaml b/salt/elasticfleet/integration-defaults.yaml new file mode 100644 index 000000000..98bbd13b7 --- /dev/null +++ b/salt/elasticfleet/integration-defaults.yaml @@ -0,0 +1,46 @@ +so-logs-INTPLACEHOLDER_x_COMPLACEHOLDER: + index_sorting: False + index_template: + composed_of: + - "logs-INTPLACEHOLDER.COMPLACEHOLDER@package" + - "logs-INTPLACEHOLDER.COMPLACEHOLDER@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_COMPLACEHOLDER_templates: + - "logs-INTPLACEHOLDER.COMPLACEHOLDER@custom" + index_patterns: + - "logs-INTPLACEHOLDER.COMPLACEHOLDER-*" + priority: 501 + template: + settings: + index: + lifecycle: + name: "so-logs-INTPLACEHOLDER.COMPLACEHOLDER-logs" + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: "60d" + delete: + actions: + delete: {} + min_age: "365d" + hot: + actions: + rollover: + max_age: "30d" + max_primary_shard_size: "50gb" + set_priority: + priority: 100 + min_age: "0ms" + warm: + actions: + set_priority: + priority: 50 + min_age: "30d" \ No newline at end of file diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 0b32628ea..7ca59401f 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -40,6 +40,11 @@ elasticfleet: global: True helpLink: elastic-fleet.html advanced: True + subscription_integrations: + description: Enable the installation of integrations that require an Elastic license. + global: True + forcedType: bool + helpLink: elastic-fleet.html server: custom_fqdn: description: Custom FQDN for Agents to connect to. One per line. diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 296e578fc..7e1e4b790 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -97,11 +97,20 @@ elastic_fleet_package_install() { curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" } +elastic_fleet_bulk_package_install() { + BULK_PKG_LIST=$1 + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk" +} + elastic_fleet_package_is_installed() { PACKAGE=$1 curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status' } +elastic_fleet_installed_packages() { + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=300" +} + elastic_fleet_agent_policy_ids() { curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id if [ $? -ne 0 ]; then diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list index 7e68c6e83..a52920a42 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list @@ -10,6 +10,6 @@ SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # List configured package policies -curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages" -H 'kbn-xsrf: true' | jq +curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages?prerelease=true" -H 'kbn-xsrf: true' | jq echo diff --git a/salt/elasticsearch/integration-templates.map.jinja b/salt/elasticsearch/integration-templates.map.jinja new file mode 100644 index 000000000..59a9222c5 --- /dev/null +++ b/salt/elasticsearch/integration-templates.map.jinja @@ -0,0 +1,110 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{% set packages = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set INTEGRATION_INDEX_SETTINGS = {} %} + + +{% set default_settings = { + 'index_sorting': false, + 'index_template': { + 'data_stream': { + 'allow_custom_routing': false, + 'hidden': false + }, + 'priority': 501, + 'template': { + 'settings': { + 'index': { + 'number_of_replicas': 0 + } + } + } + }, + 'policy': { + 'phases': { + 'cold': { + 'actions': { + 'set_priority': { + 'priority': 0 + } + }, + 'min_age': '60d' + }, + 'delete': { + 'actions': { + 'delete': {} + }, + 'min_age': '365d' + }, + 'hot': { + 'actions': { + 'rollover':{ + 'max_age': '30d', + 'max_primary_shard_size': '50gb' + }, + 'set_priority': { + 'priority': 100 + } + }, + 'min_age': '0ms' + }, + 'warm': { + 'actions': { + 'set_priority': { + 'priority': 50 + } + }, + 'min_age': '30d' + } + } + } +} %} + +{# Create template for each package component from elasticfleet/defaults.yaml #} +{% for package in packages %} + {% for pkg_name, components in package.items() %} + {% if components is not none %} + {% for component in components %} + {% set component_dot = component.replace('_x_', '.') %} + {% set template_name = 'so-logs-' ~ component %} + + {% set template = { + 'index_sorting': default_settings.index_sorting, + 'index_template': { + 'composed_of': [ + 'logs-' ~ component_dot ~ '@package', + 'logs-' ~ component_dot ~ '@custom', + 'so-fleet-_globals-1', + 'so-fleet_agent_id_verification-1' + ], + 'data_stream': default_settings.index_template.data_stream, + 'ignore_missing_component_templates': [ + 'logs-' ~ component_dot ~ '@custom' + ], + 'index_patterns': [ + 'logs-' ~ component_dot ~ '-*' + ], + 'priority': default_settings.index_template.priority, + 'template': { + 'settings': { + 'index': { + 'lifecycle': { + 'name': 'so-logs-' ~ component_dot ~ '-logs' + }, + 'number_of_replicas': default_settings.index_template.template.settings.index.number_of_replicas + } + } + } + }, + 'policy': default_settings.policy + } %} + + {% do INTEGRATION_INDEX_SETTINGS.update({template_name: template}) %} + {% endfor %} + {% endif %} + {% endfor %} +{% endfor %} \ No newline at end of file diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 507ea533d..c53349f18 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -14,6 +14,15 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} +{# start generation of integration default index_settings #} +{% if salt['file.file_exists']('/opt/so/state/estemplates.txt') %} +{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} +{% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} +{% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %} +{% endfor %} +{% endif %} +{# end generation of integration default index_settings #} + {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %} {% for index in ES_INDEX_SETTINGS_ORIG.keys() %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}