Merge pull request #10655 from Security-Onion-Solutions/feature/supported_integrations

Restructure Elasticsearch templates for supported integrations

salt/elasticfleet/defaults.yaml

@@ -23,3 +23,11 @@ elasticfleet:
         - stats
         - stderr
         - stdout
+  packages:
+    - aws
+    - azure
+    - cloudflare
+    - fim
+    - github
+    - google_workspace
+    - 1password

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -51,6 +51,21 @@ elastic_fleet_integration_update() {
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 }
 
+elastic_fleet_package_version_check() {
+    PACKAGE=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
+}
+
+elastic_fleet_package_install() {
+    PKGKEY=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PKGKEY"
+}
+
+elastic_fleet_package_is_installed() {
+    PACKAGE=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
+}
+
 elastic_fleet_policy_create() {
 
     NAME=$1

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+
+. /usr/sbin/so-elastic-fleet-common
+
+{%- for PACKAGE in SUPPORTED_PACKAGES %}
+echo "Setting up {{ PACKAGE }} package..."
+VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}")
+elastic_fleet_package_install "{{ PACKAGE }}-$VERSION"
+echo
+{%- endfor %}
+echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -48,6 +48,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 printf "\n\n"
 
 ### Create Policies & Associated Integration Configuration ###
+# Load packages
+/usr/sbin/so-elastic-fleet-package-load
+
+# Load Elasticsearch templates
+/usr/sbin/so-elasticsearch-templates-load
 
 # Manager Fleet Server Host
 elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120"

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -1,63 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-default_conf_dir=/opt/so/conf
-
-# Define a default directory to load pipelines from
-ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
-
-# Wait for ElasticSearch to initialize
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 240 ]]; do
-      so-elasticsearch-query / -k --output /dev/null --silent --head --fail
-	if [ $? -eq 0 ]; then
-		ELASTICSEARCH_CONNECTED="yes"
-		echo "connected!"
-		break
-	else
-		((COUNT+=1))
-		sleep 1
-		echo -n "."
-	fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-	echo
-	echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-	echo
-	exit 1
-fi
-
-set -e
-
-cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
-
-echo "Loading ECS component templates..."
-for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done
-
-cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
-
-echo "Loading Elastic Agent component templates..."
-for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
-
-# Load SO-specific component templates
-cd ${ELASTICSEARCH_TEMPLATES}/component/so
-
-echo "Loading Security Onion component templates..."
-for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
-echo
-
-# Load SO index templates
-cd ${ELASTICSEARCH_TEMPLATES}/index
-
-echo "Loading Security Onion index templates..."
-for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
-echo
-
-cd - >/dev/null

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+
+. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
+
+
+default_conf_dir=/opt/so/conf
+
+# Define a default directory to load pipelines from
+ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
+
+if [ -f /usr/sbin/so-elastic-fleet-common ]; then
+	# Wait for ElasticSearch to initialize
+	echo -n "Waiting for ElasticSearch..."
+	COUNT=0
+	ELASTICSEARCH_CONNECTED="no"
+	while [[ "$COUNT" -le 240 ]]; do
+      		so-elasticsearch-query / -k --output /dev/null --silent --head --fail
+		if [ $? -eq 0 ]; then
+			ELASTICSEARCH_CONNECTED="yes"
+			echo "connected!"
+			break
+		else
+			((COUNT+=1))
+			sleep 1
+			echo -n "."
+		fi
+	done
+	if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+		echo
+		echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+		echo
+		exit 1
+	fi
+
+	SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+	INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} )
+	if [ "$INSTALLED" != "installed" ]; then
+        	echo
+        	echo "Packages not yet installed."
+        	echo
+        	exit 0
+	fi
+
+	set -e
+
+	cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
+
+	echo "Loading ECS component templates..."
+	for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done
+
+	cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
+
+	echo "Loading Elastic Agent component templates..."
+	for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
+
+	# Load SO-specific component templates
+	cd ${ELASTICSEARCH_TEMPLATES}/component/so
+
+	echo "Loading Security Onion component templates..."
+	for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
+	echo
+
+	# Load SO index templates
+	cd ${ELASTICSEARCH_TEMPLATES}/index
+
+	echo "Loading Security Onion index templates..."
+	for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
+	echo
+else
+	echo "Elastic Fleet not configured. Exiting..."
+	exit 0
+fi
+	cd - >/dev/null

setup/so-functions

@@ -1149,45 +1149,7 @@ elasticsearch_pillar() {
         "      query:"\
         "        bool:"\
         "          max_clause_count: 3500"\
-		"  index_settings:"\ > $elasticsearch_pillar_file
+                "  index_settings: {}" > $elasticsearch_pillar_file
-		for INDEX in aws azure barracuda beats bluecoat cef checkpoint cisco cyberark cylance elasticsearch endgame f5 firewall fortinet gcp google_workspace imperva infoblox juniper kibana logstash microsoft misp netflow netscout o365 okta osquery proofpoint radware redis snort snyk sonicwall sophos strelka syslog tomcat zeek zscaler
-		do
-			printf '%s\n'\
-				"    so-$INDEX:"\
-        		"      warm: 7"\
-        		"      close: 30"\
-        		"      delete: 365"\
-				"      index_sorting: False"\
-				"      index_template:"\
-        		"        template:"\
-				"          settings:"\
-        		"            index:"\
-        		"              mapping:"\
-        		"                total_fields:"\
-        		"                  limit: 5000"\
-        		"              refresh_interval: 30s"\
-        		"              number_of_shards: 1"\
-        		"              number_of_replicas: 0" >> $elasticsearch_pillar_file
-		done
-		for INDEX in import
-		do
-			printf '%s\n'\
-				"    so-$INDEX:"\
-        		"      warm: 7"\
-        		"      close: 73000"\
-        		"      delete: 73001"\
-				"      index_sorting: False"\
-				"      index_template:"\
-        		"        template:"\
-				"          settings:"\
-        		"            index:"\
-        		"              mapping:"\
-        		"                total_fields:"\
-        		"                  limit: 5000"\
-        		"              refresh_interval: 30s"\
-        		"              number_of_shards: 1"\
-        		"              number_of_replicas: 0" >> $elasticsearch_pillar_file
-		done
 }
 
 es_heapsize() {