mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-06-14 14:18:40 +02:00
fixes addon integration map file
Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
This commit is contained in:
reyesj2
@@ -10,6 +10,44 @@
|
||||
{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
|
||||
{% set ADDON_INTEGRATION_DEFAULTS = {} %}
|
||||
|
||||
{# Some fleet integrations don't follow the standard naming convention #}
|
||||
{% set WEIRD_INTEGRATIONS = {
|
||||
'awsfirehose.logs': 'awsfirehose',
|
||||
'cribl.logs': 'cribl',
|
||||
'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
|
||||
'azure_application_insights.app_insights': 'azure.app_insights',
|
||||
'azure_application_insights.app_state': 'azure.app_state',
|
||||
'azure_billing.billing': 'azure.billing',
|
||||
'azure_functions.metrics': 'azure.function',
|
||||
'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
|
||||
'azure_metrics.compute_vm': 'azure.compute_vm',
|
||||
'azure_metrics.container_instance': 'azure.container_instance',
|
||||
'azure_metrics.container_registry': 'azure.container_registry',
|
||||
'azure_metrics.container_service': 'azure.container_service',
|
||||
'azure_metrics.database_account': 'azure.database_account',
|
||||
'azure_metrics.monitor': 'azure.monitor',
|
||||
'azure_metrics.storage_account': 'azure.storage_account',
|
||||
'azure_openai.metrics': 'azure.open_ai',
|
||||
'beat.state': 'beats.stack_monitoring.state',
|
||||
'beat.stats': 'beats.stack_monitoring.stats',
|
||||
'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
|
||||
'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
|
||||
'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
|
||||
'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
|
||||
'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
|
||||
'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
|
||||
'kibana.stats': 'kibana.stack_monitoring.stats',
|
||||
'kibana.status': 'kibana.stack_monitoring.status',
|
||||
'logstash.node_cel': 'logstash.stack_monitoring.node',
|
||||
'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
|
||||
'synthetics.browser': 'synthetics-browser',
|
||||
'synthetics.browser_network': 'synthetics-browser.network',
|
||||
'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
|
||||
'synthetics.http': 'synthetics-http',
|
||||
'synthetics.icmp': 'synthetics-icmp',
|
||||
'synthetics.tcp': 'synthetics-tcp'
|
||||
} %}
|
||||
|
||||
{% for pkg in ADDON_PACKAGE_COMPONENTS %}
|
||||
{% if pkg.name in CORE_ESFLEET_PACKAGES %}
|
||||
{# skip core integrations #}
|
||||
@@ -17,22 +55,36 @@
|
||||
{# generate defaults for each integration #}
|
||||
{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
|
||||
{% for pattern in pkg.es_index_patterns %}
|
||||
{% set integration_key = "so-logs-" ~ pkg.name ~ "_x_" ~ pattern.title %}
|
||||
{% set integration_defaults = {
|
||||
{% if "metrics-" in pattern.name %}
|
||||
{% set integration_type = "metrics-" %}
|
||||
{% elif "logs-" in pattern.name %}
|
||||
{% set integration_type = "logs-" %}
|
||||
{% else %}
|
||||
{% set integration_type = "" %}
|
||||
{% endif %}
|
||||
{% set component_name = pkg.name ~ "." ~ pattern.title %}
|
||||
{# fix weirdly named components #}
|
||||
{% if component_name in WEIRD_INTEGRATIONS %}
|
||||
{% set component_name = WEIRD_INTEGRATIONS[component_name] %}
|
||||
{% endif %}
|
||||
{% set integration_key = "so-" ~ integration_type ~ component_name %}
|
||||
|
||||
{# Default integration settings #}
|
||||
{% set integration_defaults = {
|
||||
"index_sorting": false,
|
||||
"index_template": {
|
||||
"composed_of": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@package", "logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
|
||||
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
|
||||
"data_stream": {
|
||||
"hidden": false,
|
||||
"allow_custom_routing": false
|
||||
"allow_custom_routing": false,
|
||||
"hidden": false
|
||||
},
|
||||
"ignore_missing_component_templates": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom"],
|
||||
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
|
||||
"index_patterns": [pattern.name],
|
||||
"priority": 501,
|
||||
"template": {
|
||||
"settings": {
|
||||
"index": {
|
||||
"lifecycle": {"name": "so-logs-" ~ pkg.name ~ "." ~ pattern.title ~ "-logs"},
|
||||
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
|
||||
"number_of_replicas": 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
|
||||
# this file except in compliance with the Elastic License 2.0.
|
||||
|
||||
. /usr/sbin/so-common
|
||||
. /usr/sbin/so-elastic-fleet-common
|
||||
|
||||
# Check that /opt/so/state/estemplates.txt exists to signal that Elasticsearch
|
||||
# has completed its first run of core-only integrations/indices/components/ilm
|
||||
STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
|
||||
INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
|
||||
BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
|
||||
BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
|
||||
PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
|
||||
|
||||
SKIP_SUBSCRIPTION=true
|
||||
PENDING_UPDATE=false
|
||||
|
||||
version_conversion(){
|
||||
version=$1
|
||||
echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
|
||||
}
|
||||
|
||||
compare_versions() {
|
||||
version1=$1
|
||||
version2=$2
|
||||
|
||||
# Convert versions to numbers
|
||||
num1=$(version_conversion "$version1")
|
||||
num2=$(version_conversion "$version2")
|
||||
|
||||
# Compare using bc
|
||||
if (( $(echo "$num1 < $num2" | bc -l) )); then
|
||||
echo "less"
|
||||
elif (( $(echo "$num1 > $num2" | bc -l) )); then
|
||||
echo "greater"
|
||||
else
|
||||
echo "equal"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f $STATE_FILE_SUCCESS ]]; then
|
||||
if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then
|
||||
# Package_list contains all NON-beta integrations.
|
||||
latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list)
|
||||
echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST
|
||||
rm -f $INSTALLED_PACKAGE_LIST
|
||||
echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
|
||||
|
||||
cat "$INSTALLED_PACKAGE_LIST" | jq -c '.packages[]' | while read -r package; do
|
||||
# get package details
|
||||
package_name=$(echo "$package" | jq -r '.name')
|
||||
latest_version=$(echo "$package" | jq -r '.latest_version')
|
||||
installed_version=$(echo "$package" | jq -r '.installed_version')
|
||||
subscription=$(echo "$package" | jq -r '.subscription')
|
||||
bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
|
||||
|
||||
if [ $SKIP_SUBSCRIPTION ] && [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
|
||||
# pass over integrations that require non-basic elastic license
|
||||
continue
|
||||
else
|
||||
if [ -n "$installed_version" ]; then
|
||||
results=$(compare_versions "$latest_version" "$installed_version")
|
||||
if [ $results == "greater" ]; then
|
||||
echo "$package_name is not up to date... Adding to next update."
|
||||
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
|
||||
PENDING_UPDATE=true
|
||||
fi
|
||||
else
|
||||
echo "$package_name is not installed... Adding to next update."
|
||||
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
|
||||
PENDING_UPDATE=true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $PENDING_UPDATE ]; then
|
||||
# Run bulk install of packages
|
||||
elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST
|
||||
|
||||
# Write out file for generating index/component/ilm templates
|
||||
latest_installed_package_list=$(elastic_fleet_installed_packages)
|
||||
echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
|
||||
|
||||
else
|
||||
echo "Elastic integrations don't appear to need installation/updating..."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
else
|
||||
# This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.
|
||||
echo "Elastic Fleet does not appear to be responding... Exiting... "
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
# This message will appear when an update to core integration is made and this script is run at the same time as
|
||||
# elasticsearch.enabled -> detects change to core index settings -> deletes estemplates.txt
|
||||
echo "Elasticsearch may not be fully configured yet or is currently updating core index settings."
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user