mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 17:22:49 +01:00
Merge pull request #10655 from Security-Onion-Solutions/feature/supported_integrations
Restructure Elasticsearch templates for supported integrations
This commit is contained in:
weslambert
GitHub
@@ -23,3 +23,11 @@ elasticfleet:
|
||||
- stats
|
||||
- stderr
|
||||
- stdout
|
||||
packages:
|
||||
- aws
|
||||
- azure
|
||||
- cloudflare
|
||||
- fim
|
||||
- github
|
||||
- google_workspace
|
||||
- 1password
|
||||
|
||||
@@ -51,6 +51,21 @@ elastic_fleet_integration_update() {
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
|
||||
}
|
||||
|
||||
elastic_fleet_package_version_check() {
|
||||
PACKAGE=$1
|
||||
curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
|
||||
}
|
||||
|
||||
elastic_fleet_package_install() {
|
||||
PKGKEY=$1
|
||||
curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PKGKEY"
|
||||
}
|
||||
|
||||
elastic_fleet_package_is_installed() {
|
||||
PACKAGE=$1
|
||||
curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
|
||||
}
|
||||
|
||||
elastic_fleet_policy_create() {
|
||||
|
||||
NAME=$1
|
||||
|
||||
17
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
Executable file
17
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
Executable file
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
|
||||
# this file except in compliance with the Elastic License 2.0.
|
||||
{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
|
||||
{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
|
||||
|
||||
. /usr/sbin/so-elastic-fleet-common
|
||||
|
||||
{%- for PACKAGE in SUPPORTED_PACKAGES %}
|
||||
echo "Setting up {{ PACKAGE }} package..."
|
||||
VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}")
|
||||
elastic_fleet_package_install "{{ PACKAGE }}-$VERSION"
|
||||
echo
|
||||
{%- endfor %}
|
||||
echo
|
||||
@@ -48,6 +48,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
|
||||
printf "\n\n"
|
||||
|
||||
### Create Policies & Associated Integration Configuration ###
|
||||
# Load packages
|
||||
/usr/sbin/so-elastic-fleet-package-load
|
||||
|
||||
# Load Elasticsearch templates
|
||||
/usr/sbin/so-elasticsearch-templates-load
|
||||
|
||||
# Manager Fleet Server Host
|
||||
elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,63 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
default_conf_dir=/opt/so/conf
|
||||
|
||||
# Define a default directory to load pipelines from
|
||||
ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
|
||||
|
||||
# Wait for ElasticSearch to initialize
|
||||
echo -n "Waiting for ElasticSearch..."
|
||||
COUNT=0
|
||||
ELASTICSEARCH_CONNECTED="no"
|
||||
while [[ "$COUNT" -le 240 ]]; do
|
||||
so-elasticsearch-query / -k --output /dev/null --silent --head --fail
|
||||
if [ $? -eq 0 ]; then
|
||||
ELASTICSEARCH_CONNECTED="yes"
|
||||
echo "connected!"
|
||||
break
|
||||
else
|
||||
((COUNT+=1))
|
||||
sleep 1
|
||||
echo -n "."
|
||||
fi
|
||||
done
|
||||
if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
|
||||
echo
|
||||
echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
|
||||
|
||||
echo "Loading ECS component templates..."
|
||||
for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done
|
||||
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
|
||||
|
||||
echo "Loading Elastic Agent component templates..."
|
||||
for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
|
||||
|
||||
# Load SO-specific component templates
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/component/so
|
||||
|
||||
echo "Loading Security Onion component templates..."
|
||||
for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
|
||||
echo
|
||||
|
||||
# Load SO index templates
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/index
|
||||
|
||||
echo "Loading Security Onion index templates..."
|
||||
for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
|
||||
echo
|
||||
|
||||
cd - >/dev/null
|
||||
80
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
Executable file
80
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
Executable file
@@ -0,0 +1,80 @@
|
||||
#!/bin/bash
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
|
||||
{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
|
||||
|
||||
. /usr/sbin/so-common
|
||||
. /usr/sbin/so-elastic-fleet-common
|
||||
|
||||
|
||||
default_conf_dir=/opt/so/conf
|
||||
|
||||
# Define a default directory to load pipelines from
|
||||
ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
|
||||
|
||||
if [ -f /usr/sbin/so-elastic-fleet-common ]; then
|
||||
# Wait for ElasticSearch to initialize
|
||||
echo -n "Waiting for ElasticSearch..."
|
||||
COUNT=0
|
||||
ELASTICSEARCH_CONNECTED="no"
|
||||
while [[ "$COUNT" -le 240 ]]; do
|
||||
so-elasticsearch-query / -k --output /dev/null --silent --head --fail
|
||||
if [ $? -eq 0 ]; then
|
||||
ELASTICSEARCH_CONNECTED="yes"
|
||||
echo "connected!"
|
||||
break
|
||||
else
|
||||
((COUNT+=1))
|
||||
sleep 1
|
||||
echo -n "."
|
||||
fi
|
||||
done
|
||||
if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
|
||||
echo
|
||||
echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
|
||||
INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} )
|
||||
if [ "$INSTALLED" != "installed" ]; then
|
||||
echo
|
||||
echo "Packages not yet installed."
|
||||
echo
|
||||
exit 0
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
|
||||
|
||||
echo "Loading ECS component templates..."
|
||||
for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done
|
||||
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
|
||||
|
||||
echo "Loading Elastic Agent component templates..."
|
||||
for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
|
||||
|
||||
# Load SO-specific component templates
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/component/so
|
||||
|
||||
echo "Loading Security Onion component templates..."
|
||||
for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
|
||||
echo
|
||||
|
||||
# Load SO index templates
|
||||
cd ${ELASTICSEARCH_TEMPLATES}/index
|
||||
|
||||
echo "Loading Security Onion index templates..."
|
||||
for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
|
||||
echo
|
||||
else
|
||||
echo "Elastic Fleet not configured. Exiting..."
|
||||
exit 0
|
||||
fi
|
||||
cd - >/dev/null
|
||||
@@ -1149,45 +1149,7 @@ elasticsearch_pillar() {
|
||||
" query:"\
|
||||
" bool:"\
|
||||
" max_clause_count: 3500"\
|
||||
" index_settings:"\ > $elasticsearch_pillar_file
|
||||
for INDEX in aws azure barracuda beats bluecoat cef checkpoint cisco cyberark cylance elasticsearch endgame f5 firewall fortinet gcp google_workspace imperva infoblox juniper kibana logstash microsoft misp netflow netscout o365 okta osquery proofpoint radware redis snort snyk sonicwall sophos strelka syslog tomcat zeek zscaler
|
||||
do
|
||||
printf '%s\n'\
|
||||
" so-$INDEX:"\
|
||||
" warm: 7"\
|
||||
" close: 30"\
|
||||
" delete: 365"\
|
||||
" index_sorting: False"\
|
||||
" index_template:"\
|
||||
" template:"\
|
||||
" settings:"\
|
||||
" index:"\
|
||||
" mapping:"\
|
||||
" total_fields:"\
|
||||
" limit: 5000"\
|
||||
" refresh_interval: 30s"\
|
||||
" number_of_shards: 1"\
|
||||
" number_of_replicas: 0" >> $elasticsearch_pillar_file
|
||||
done
|
||||
for INDEX in import
|
||||
do
|
||||
printf '%s\n'\
|
||||
" so-$INDEX:"\
|
||||
" warm: 7"\
|
||||
" close: 73000"\
|
||||
" delete: 73001"\
|
||||
" index_sorting: False"\
|
||||
" index_template:"\
|
||||
" template:"\
|
||||
" settings:"\
|
||||
" index:"\
|
||||
" mapping:"\
|
||||
" total_fields:"\
|
||||
" limit: 5000"\
|
||||
" refresh_interval: 30s"\
|
||||
" number_of_shards: 1"\
|
||||
" number_of_replicas: 0" >> $elasticsearch_pillar_file
|
||||
done
|
||||
" index_settings: {}" > $elasticsearch_pillar_file
|
||||
}
|
||||
|
||||
es_heapsize() {
|
||||
|
||||
Reference in New Issue
Block a user