mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-07-14 04:43:09 +02:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Jason Ertel
|
6fc0fd954c | ||
|
Jason Ertel
|
566f90a0c0 |
@@ -35,9 +35,6 @@ case $1 in
|
||||
"elastic-fleet"|"elasticfleet")
|
||||
docker_check_running "elastic-fleet" "--stop"
|
||||
docker rm "so-elastic-fleet" 2> /dev/null
|
||||
# Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
|
||||
rm -rf /opt/so/conf/elastic-fleet/state
|
||||
|
||||
salt-call state.apply elasticfleet queue=True
|
||||
;;
|
||||
*)
|
||||
|
||||
@@ -29,8 +29,6 @@ case $1 in
|
||||
"elasticfleet"|"elastic-fleet")
|
||||
docker_check_running "elastic-fleet" "--stop"
|
||||
docker rm "so-elastic-fleet" 2> /dev/null
|
||||
# Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
|
||||
rm -rf /opt/so/conf/elastic-fleet/state
|
||||
;;
|
||||
*)
|
||||
docker_check_running "$1" "--stop"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
elasticfleet:
|
||||
enabled: False
|
||||
patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
|
||||
enable_manager_output: True
|
||||
config:
|
||||
server:
|
||||
|
||||
@@ -11,10 +11,6 @@
|
||||
|
||||
{# This value is generated during node install and stored in minion pillar #}
|
||||
{% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
|
||||
{# Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
|
||||
- if a fresh enrollment is needed use 'so-stop elasticfleet'
|
||||
#}
|
||||
{% set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
|
||||
|
||||
include:
|
||||
- ca
|
||||
@@ -69,7 +65,6 @@ so-elastic-fleet:
|
||||
- /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
|
||||
- /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
|
||||
- /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
|
||||
- /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
|
||||
- /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs
|
||||
{% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
|
||||
{% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
|
||||
@@ -77,7 +72,6 @@ so-elastic-fleet:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- environment:
|
||||
{% if not ENROLLED %}
|
||||
- FLEET_SERVER_ENABLE=true
|
||||
- FLEET_URL=https://{{ GLOBALS.hostname }}:8220
|
||||
- FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
|
||||
@@ -87,9 +81,6 @@ so-elastic-fleet:
|
||||
- FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
|
||||
- FLEET_CA=/etc/pki/tls/certs/intca.crt
|
||||
- FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
|
||||
{% endif %}
|
||||
- STATE_PATH=/usr/share/elastic-agent/state
|
||||
- CONFIG_PATH=/usr/share/elastic-agent/state
|
||||
- LOGS_PATH=logs
|
||||
{% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
|
||||
{% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
|
||||
@@ -108,7 +99,6 @@ so-elastic-fleet:
|
||||
- x509: etc_elasticfleet_crt
|
||||
- require:
|
||||
- file: trusttheca
|
||||
- file: eastatedir
|
||||
- x509: etc_elasticfleet_key
|
||||
- x509: etc_elasticfleet_crt
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"package": {
|
||||
"name": "endpoint",
|
||||
"title": "Elastic Defend",
|
||||
"version": "9.3.1",
|
||||
"version": "9.3.0",
|
||||
"requires_root": true
|
||||
},
|
||||
"enabled": true,
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"\\.gz$"
|
||||
],
|
||||
"include_files": [],
|
||||
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
|
||||
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
|
||||
"tags": [
|
||||
"import"
|
||||
],
|
||||
|
||||
@@ -10,15 +10,6 @@
|
||||
{% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
|
||||
{% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
|
||||
|
||||
so-elastic-agent-install:
|
||||
file.managed:
|
||||
- name: /usr/sbin/so-elastic-agent-install
|
||||
- source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
|
||||
- user: 947
|
||||
- group: 939
|
||||
- mode: 755
|
||||
- show_changes: False
|
||||
|
||||
{% if not AGENT_STATUS or not AGENT_EXISTS %}
|
||||
|
||||
pull_agent_installer:
|
||||
@@ -30,9 +21,11 @@ pull_agent_installer:
|
||||
|
||||
run_installer:
|
||||
cmd.run:
|
||||
- name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}"
|
||||
- require:
|
||||
- file: pull_agent_installer
|
||||
- name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
|
||||
- cwd: /opt/so
|
||||
- retry:
|
||||
attempts: 3
|
||||
interval: 20
|
||||
|
||||
cleanup_agent_installer:
|
||||
file.absent:
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
. /usr/sbin/so-elastic-fleet-common
|
||||
|
||||
|
||||
# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later
|
||||
GRIDNODETOKEN="$1"
|
||||
LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log"
|
||||
|
||||
check_agent_health() {
|
||||
timeout=300
|
||||
interval=10
|
||||
start=$SECONDS
|
||||
|
||||
while (( SECONDS - start < timeout )); do
|
||||
agent_status=$(elastic-agent status 2>&1)
|
||||
echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
|
||||
if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
|
||||
return 0
|
||||
fi
|
||||
echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..."
|
||||
sleep "$interval"
|
||||
done
|
||||
|
||||
echo "The Elastic Agent did not become healthy within ${timeout} seconds"
|
||||
return 1
|
||||
}
|
||||
|
||||
uninstall_agent() {
|
||||
if command -v elastic-agent >/dev/null 2>&1; then
|
||||
elastic-agent uninstall -f
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -z "$GRIDNODETOKEN" ]]; then
|
||||
noderole=$(so-yaml.py get -r /etc/salt/grains role)
|
||||
if [[ "$noderole" == "so-heavynode" ]]; then
|
||||
GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only)
|
||||
else
|
||||
GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only)
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$GRIDNODETOKEN" ]]; then
|
||||
echo "Unable to determine Elastic Fleet enrollment token. Exiting."
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
|
||||
echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running."
|
||||
|
||||
# When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently.
|
||||
# PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call:
|
||||
# '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! <salt.transport.zeromq.RequestClient object at 0x7fc5f0ee7a30>
|
||||
# File "/bin/salt-call", line 12, in <module>
|
||||
# sys.exit(salt_call())'
|
||||
|
||||
PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True
|
||||
|
||||
fi
|
||||
|
||||
if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
|
||||
attempts=0
|
||||
cd /opt/so/ || exit 1
|
||||
|
||||
truncate -s 0 "$LOGFILE"
|
||||
|
||||
uninstall_agent
|
||||
|
||||
while [[ $attempts -lt 3 ]]; do
|
||||
if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then
|
||||
rm -f /opt/so/so-elastic-agent_linux_amd64
|
||||
elastic-agent status
|
||||
|
||||
exit 0
|
||||
fi
|
||||
|
||||
attempts=$((attempts + 1))
|
||||
|
||||
if [[ $attempts -lt 3 ]]; then
|
||||
echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..."
|
||||
sleep 20
|
||||
fi
|
||||
done
|
||||
|
||||
uninstall_agent
|
||||
rm -f /opt/so/so-elastic-agent_linux_amd64
|
||||
echo "The so-elastic-agent installer failed after 3 attempts. Exiting."
|
||||
|
||||
exit 1
|
||||
else
|
||||
echo "Unable to locate so-elastic-agent installer. Exiting."
|
||||
|
||||
exit 1
|
||||
fi
|
||||
@@ -30,7 +30,7 @@ done
|
||||
if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then
|
||||
printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
|
||||
printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
|
||||
exit 1
|
||||
exit
|
||||
fi
|
||||
|
||||
OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
|
||||
@@ -62,54 +62,31 @@ do
|
||||
done
|
||||
|
||||
GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
|
||||
GOARCH="amd64"
|
||||
printf "\n### Generating OS packages using the cleaned up tarballs"
|
||||
for GOOS in "${GOTARGETOS[@]}"; do
|
||||
GOARCH="amd64"
|
||||
for GOOS in "${GOTARGETOS[@]}"
|
||||
do
|
||||
if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
|
||||
printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
|
||||
docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
|
||||
--mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
|
||||
--mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
|
||||
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
|
||||
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
|
||||
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH}
|
||||
printf "\n### $GOOS/$GOARCH Installer Generated...\n"
|
||||
done
|
||||
|
||||
printf "\n\n### Generating MSI...\n"
|
||||
cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
|
||||
cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
|
||||
docker run \
|
||||
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
|
||||
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
|
||||
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
|
||||
printf "\n### MSI Generated...\n"
|
||||
|
||||
# Verify installers were created
|
||||
for GOOS in "${GOTARGETOS[@]}"; do
|
||||
GOARCH="amd64"
|
||||
if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi
|
||||
if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then
|
||||
printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH"
|
||||
exit 1
|
||||
fi
|
||||
# After verifying new installer was generated, move it to so_agent-installers directory
|
||||
mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
|
||||
done
|
||||
|
||||
# Verify MSI installer
|
||||
if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then
|
||||
printf "\n### ERROR: Installer MSI was not generated. Exiting...\n"
|
||||
exit 1
|
||||
else
|
||||
# After verifying new installer MSI was generated, move it to so_agent-installers directory
|
||||
mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
|
||||
fi
|
||||
|
||||
printf "\n### Cleaning up temp files \n"
|
||||
rm -rf /nsm/elastic-agent-workspace
|
||||
rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
|
||||
rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
|
||||
|
||||
printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
|
||||
\cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
|
||||
chmod 644 /nsm/elastic-fleet/so_agent-installers/*
|
||||
|
||||
# if we got here all installers have been generated successfully
|
||||
exit 0
|
||||
|
||||
@@ -244,34 +244,11 @@ printf '%s\n'\
|
||||
"" >> "$global_pillar_file"
|
||||
|
||||
# Call Elastic-Fleet Salt State
|
||||
printf "\nApplying elasticfleet state\n"
|
||||
for state_attempt in {1..3}; do
|
||||
if salt-call state.apply elasticfleet queue=True; then
|
||||
break
|
||||
elif [[ $state_attempt -lt 3 ]]; then
|
||||
printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt"
|
||||
sleep 10
|
||||
else
|
||||
printf "\nFailure(s) in elasticfleet state... Exiting...\n"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
printf "\nApplying elasticfleet state"
|
||||
salt-call state.apply elasticfleet queue=True
|
||||
|
||||
# Generate installers & install Elastic Agent on the node
|
||||
for agent_gen_attempt in {1..3}; do
|
||||
if so-elastic-agent-gen-installers; then
|
||||
break
|
||||
elif [[ $agent_gen_attempt -lt 3 ]]; then
|
||||
printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt"
|
||||
sleep 10
|
||||
else
|
||||
printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
printf "\nApplying elasticfleet.install_agent_grid state\n"
|
||||
if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
|
||||
printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
|
||||
exit 1
|
||||
fi
|
||||
so-elastic-agent-gen-installers
|
||||
printf "\nApplying elasticfleet.install_agent_grid state"
|
||||
salt-call state.apply elasticfleet.install_agent_grid queue=True
|
||||
exit 0
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
elasticsearch:
|
||||
enabled: false
|
||||
version: 9.3.7
|
||||
version: 9.3.3
|
||||
index_clean: true
|
||||
data_retention_method: DLM
|
||||
vm:
|
||||
|
||||
+10
-10
@@ -118,70 +118,70 @@
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_e16851a7",
|
||||
"name": "logs-pfsense.log-1.25.4-firewall",
|
||||
"name": "logs-pfsense.log-1.25.2-firewall",
|
||||
"if": "ctx.event.provider == 'filterlog'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_828590b5",
|
||||
"name": "logs-pfsense.log-1.25.4-openvpn",
|
||||
"name": "logs-pfsense.log-1.25.2-openvpn",
|
||||
"if": "ctx.event.provider == 'openvpn'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_9d37039c",
|
||||
"name": "logs-pfsense.log-1.25.4-ipsec",
|
||||
"name": "logs-pfsense.log-1.25.2-ipsec",
|
||||
"if": "ctx.event.provider == 'charon'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_ad56bbca",
|
||||
"name": "logs-pfsense.log-1.25.4-dhcp",
|
||||
"name": "logs-pfsense.log-1.25.2-dhcp",
|
||||
"if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_dd85553d",
|
||||
"name": "logs-pfsense.log-1.25.4-unbound",
|
||||
"name": "logs-pfsense.log-1.25.2-unbound",
|
||||
"if": "ctx.event.provider == 'unbound'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_720ed255",
|
||||
"name": "logs-pfsense.log-1.25.4-haproxy",
|
||||
"name": "logs-pfsense.log-1.25.2-haproxy",
|
||||
"if": "ctx.event.provider == 'haproxy'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_456beba5",
|
||||
"name": "logs-pfsense.log-1.25.4-php-fpm",
|
||||
"name": "logs-pfsense.log-1.25.2-php-fpm",
|
||||
"if": "ctx.event.provider == 'php-fpm'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_a0d89375",
|
||||
"name": "logs-pfsense.log-1.25.4-squid",
|
||||
"name": "logs-pfsense.log-1.25.2-squid",
|
||||
"if": "ctx.event.provider == 'squid'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag": "pipeline_c2f1ed55",
|
||||
"name": "logs-pfsense.log-1.25.4-snort",
|
||||
"name": "logs-pfsense.log-1.25.2-snort",
|
||||
"if": "ctx.event.provider == 'snort'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"tag":"pipeline_33db1c9e",
|
||||
"name": "logs-pfsense.log-1.25.4-suricata",
|
||||
"name": "logs-pfsense.log-1.25.2-suricata",
|
||||
"if": "ctx.event.provider == 'suricata'"
|
||||
}
|
||||
},
|
||||
@@ -28,14 +28,14 @@ elasticsearch:
|
||||
description: The maximum number of memory map areas a process may use. Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts could be too low, which may result in out of memory exceptions.
|
||||
forcedType: int
|
||||
helpLink: elasticsearch
|
||||
retention:
|
||||
retention:
|
||||
retention_pct:
|
||||
decription: Total percentage of space used by Elasticsearch for multi node clusters
|
||||
helpLink: elasticsearch
|
||||
global: True
|
||||
config:
|
||||
cluster:
|
||||
name:
|
||||
name:
|
||||
description: The name of the Security Onion Elasticsearch cluster, for identification purposes.
|
||||
readonly: True
|
||||
global: True
|
||||
@@ -55,13 +55,13 @@ elasticsearch:
|
||||
forcedType: bool
|
||||
helpLink: elasticsearch
|
||||
watermark:
|
||||
low:
|
||||
low:
|
||||
description: The lower percentage of used disk space representing a healthy node.
|
||||
helpLink: elasticsearch
|
||||
high:
|
||||
high:
|
||||
description: The higher percentage of used disk space representing an unhealthy node.
|
||||
helpLink: elasticsearch
|
||||
flood_stage:
|
||||
flood_stage:
|
||||
description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
|
||||
helpLink: elasticsearch
|
||||
action:
|
||||
@@ -172,11 +172,11 @@ elasticsearch:
|
||||
forcedType: int
|
||||
global: True
|
||||
helpLink: elasticsearch
|
||||
refresh_interval:
|
||||
refresh_interval:
|
||||
description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
|
||||
global: True
|
||||
helpLink: elasticsearch
|
||||
number_of_shards:
|
||||
number_of_shards:
|
||||
description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
|
||||
global: True
|
||||
helpLink: elasticsearch
|
||||
@@ -269,7 +269,7 @@ elasticsearch:
|
||||
global: True
|
||||
advanced: True
|
||||
warm:
|
||||
min_age:
|
||||
min_age:
|
||||
description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
|
||||
regex: ^[0-9]{1,5}d$
|
||||
forcedType: string
|
||||
@@ -370,7 +370,7 @@ elasticsearch:
|
||||
template:
|
||||
settings:
|
||||
index:
|
||||
number_of_replicas:
|
||||
number_of_replicas:
|
||||
description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
|
||||
forcedType: int
|
||||
global: True
|
||||
@@ -391,12 +391,12 @@ elasticsearch:
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: elasticsearch
|
||||
refresh_interval:
|
||||
refresh_interval:
|
||||
description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: elasticsearch
|
||||
number_of_shards:
|
||||
number_of_shards:
|
||||
description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
|
||||
global: True
|
||||
advanced: True
|
||||
@@ -645,7 +645,6 @@ elasticsearch:
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: elasticsearch
|
||||
so-logs-soc: *dataStreamSettings
|
||||
so-logs-system_x_auth: *dataStreamSettings
|
||||
so-logs-system_x_syslog: *dataStreamSettings
|
||||
so-logs-system_x_system: *dataStreamSettings
|
||||
|
||||
@@ -22,7 +22,7 @@ kibana:
|
||||
- default
|
||||
- file
|
||||
migrations:
|
||||
discardCorruptObjects: "9.3.7"
|
||||
discardCorruptObjects: "9.3.3"
|
||||
telemetry:
|
||||
enabled: False
|
||||
xpack:
|
||||
|
||||
@@ -740,6 +740,7 @@ fix_logstash_0013_lumberjack_pipeline_name() {
|
||||
up_to_3.1.0() {
|
||||
ensure_postgres_local_pillar
|
||||
ensure_postgres_secret
|
||||
determine_elastic_agent_upgrade
|
||||
elasticsearch_backup_index_templates
|
||||
# Clear existing component template state file.
|
||||
rm -f /opt/so/state/esfleet_component_templates.json
|
||||
@@ -887,9 +888,6 @@ update_kafka_metadata() {
|
||||
}
|
||||
|
||||
up_to_3.2.0() {
|
||||
# download 9.3.7 elastic agent packages
|
||||
determine_elastic_agent_upgrade
|
||||
|
||||
fix_logstash_0013_lumberjack_pipeline_name
|
||||
|
||||
pin_elasticsearch_data_retention_method
|
||||
@@ -903,7 +901,7 @@ post_to_3.2.0() {
|
||||
|
||||
bootstrap_so_soc_database
|
||||
|
||||
# Generate 9.3.7 elastic agent installers
|
||||
# Including agent regen script here since it was missed in post_to_3.1.0
|
||||
echo "Regenerating Elastic Agent Installers"
|
||||
/sbin/so-elastic-agent-gen-installers
|
||||
|
||||
@@ -1176,8 +1174,7 @@ verify_es_version_compatibility() {
|
||||
["8.18.4"]="8.18.6 8.18.8 9.0.8"
|
||||
["8.18.6"]="8.18.8 9.0.8"
|
||||
["8.18.8"]="9.0.8"
|
||||
["9.0.8"]="9.3.3 9.3.7"
|
||||
["9.3.3"]="9.3.7"
|
||||
["9.0.8"]="9.3.3"
|
||||
)
|
||||
|
||||
# Elasticsearch MUST upgrade through these versions
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
|
||||
{% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %}
|
||||
{% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %}
|
||||
{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
|
||||
|
||||
{% for module, application_url in GLOBALS.application_urls.items() %}
|
||||
{% do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %}
|
||||
@@ -24,13 +25,22 @@
|
||||
|
||||
{% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
|
||||
|
||||
{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
|
||||
{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
|
||||
{% for tool in SOCDEFAULTS.soc.config.server.client.tools %}
|
||||
{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %}
|
||||
{% do tool.update({'link': METRICS_LINK}) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if TELEGRAFMERGED.output == 'POSTGRES' %}
|
||||
{% for tool in SOCDEFAULTS.soc.config.server.client.tools %}
|
||||
{% if tool.name == "toolInfluxDb" %}
|
||||
{% do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% else %}
|
||||
{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
|
||||
{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
|
||||
{% for tool in SOCDEFAULTS.soc.config.server.client.tools %}
|
||||
{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %}
|
||||
{% do tool.update({'link': METRICS_LINK}) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %}
|
||||
|
||||
|
||||
@@ -7,6 +7,11 @@
|
||||
{% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
|
||||
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
|
||||
{% from 'manager/map.jinja' import MANAGERMERGED %}
|
||||
{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
|
||||
{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %}
|
||||
{%- set PG_USER = PG_ENTRY.get('user', '') %}
|
||||
{%- set PG_PASS = PG_ENTRY.get('pass', '') %}
|
||||
|
||||
{% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %}
|
||||
{% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
|
||||
|
||||
@@ -75,6 +80,20 @@
|
||||
{% do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
|
||||
{% endif %}
|
||||
|
||||
{# Define the postgresmetrics module if telegraf is setup to only use Postgres #}
|
||||
{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %}
|
||||
{% do SOCMERGED.config.server.modules.update({
|
||||
'postgresmetrics': {
|
||||
'database': 'so_telegraf',
|
||||
'host': GLOBALS.manager_ip,
|
||||
'password': PG_PASS,
|
||||
'port': 5432,
|
||||
'sslMode': 'allow',
|
||||
'user': PG_USER,
|
||||
}
|
||||
}) %}
|
||||
{% do SOCMERGED.config.server.modules.pop('influxdb') %}
|
||||
{% endif %}
|
||||
|
||||
{# Define the Detections custom ruleset that should always be present #}
|
||||
{% set CUSTOM_RULESET = {
|
||||
|
||||
@@ -5,7 +5,7 @@ telegraf:
|
||||
advanced: True
|
||||
helpLink: influxdb
|
||||
output:
|
||||
description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
|
||||
description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. When set to BOTH, the grid screen's metrics are pulled from Postgres, and the InfluxDB tool link remains visible. When set to POSTGRES, the InfluxDB tool link is removed.
|
||||
options:
|
||||
- INFLUXDB
|
||||
- POSTGRES
|
||||
|
||||
Reference in New Issue
Block a user