CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-05-22 03:01:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

add ingest latency metrics

Browse Source
This commit is contained in:
reyesj2
2026-05-11 16:51:38 -05:00
parent 49a643fff4
commit 492ae80da7
11 changed files with 162 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/logstash/defaults.yaml
+3 -2
View File
@@ -26,12 +26,12 @@ logstash:
manager:
- so/0011_input_endgame.conf
- so/0012_input_elastic_agent.conf.jinja
- so/0013_input_lumberjack_fleet.conf
- so/0013_input_lumberjack_fleet.conf.jinja
- so/9999_output_redis.conf.jinja
receiver:
- so/0011_input_endgame.conf
- so/0012_input_elastic_agent.conf.jinja
- so/0013_input_lumberjack_fleet.conf
- so/0013_input_lumberjack_fleet.conf.jinja
- so/9999_output_redis.conf.jinja
search:
- so/0900_input_redis.conf.jinja
@@ -69,4 +69,5 @@ logstash:
pipeline_x_batch_x_size: 125
pipeline_x_ecs_compatibility: disabled
dmz_nodes: []
latency_metrics: False
salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja
+10 -4
View File
@@ -1,3 +1,4 @@
{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
input {
elastic_agent {
port => 5055
@@ -11,10 +12,15 @@ input {
}
}
filter {
if ![metadata] {
mutate {
rename => {"@metadata" => "metadata"}
{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
ruby {
code => "event.set('[_tmp][logstash_from_agent]', Time.now().utc.iso8601(3));"
}
{% endif %}
if ![metadata] {
mutate {
rename => {"@metadata" => "metadata"}
}
}
}
}
salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf
-23
View File
@@ -1,23 +0,0 @@
input {
elastic_agent {
port => 5056
tags => [ "elastic-agent", "fleet-lumberjack-input" ]
ssl_enabled => true
ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
ecs_compatibility => v8
id => "fleet-lumberjack-in"
codec => "json"
}
}
filter {
if ![metadata] {
mutate {
rename => {"@metadata" => "metadata"}
}
}
}
salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja
+26
View File
@@ -0,0 +1,26 @@
{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
input {
elastic_agent {
port => 5056
tags => [ "elastic-agent", "fleet-lumberjack-input" ]
ssl_enabled => true
ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
ecs_compatibility => v8
id => "fleet-lumberjack-in"
codec => "json"
}
}
filter {
{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
ruby {
code => "event.set('[_tmp][logstash_from_fleet]', Time.now().utc.iso8601(3));"
}
{% endif %}
if ![metadata] {
mutate {
rename => {"@metadata" => "metadata"}
}
}
}
salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja
+6
View File
@@ -1,3 +1,4 @@
{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
{%- set kafka_password = salt['pillar.get']('kafka:config:password') %}
{%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %}
{%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
@@ -30,6 +31,11 @@ input {
}
}
filter {
{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
ruby {
code => "event.set('[_tmp][logstash_from_kafka]', Time.now().utc.iso8601(3));"
}
{% endif %}
if ![metadata] {
mutate {
rename => { "@metadata" => "metadata" }
salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
+8 -1
View File
@@ -1,4 +1,4 @@
{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES, LOGSTASH_MERGED %}
{%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
{%- for index in range(LOGSTASH_REDIS_NODES|length) %}
@@ -18,3 +18,10 @@ input {
}
{% endfor %}
{% endfor -%}
filter {
{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
ruby {
code => "event.set('[_tmp][logstash_from_redis]', Time.now().utc.iso8601(3));"
}
{% endif %}
}
salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja
+8
View File
@@ -1,3 +1,11 @@
{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
filter {
ruby {
code => "event.set('[_tmp][logstash_to_elasticsearch]', Time.now().utc.iso8601(3));"
}
}
{% endif %}
output {
if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] {
elasticsearch {
salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja
+13 -6
View File
@@ -13,13 +13,20 @@ filter {
add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}"
}
}
output {
lumberjack {
codec => json
{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
filter {
ruby {
code => "event.set('[_tmp][fleet_to_logstash]', Time.now().utc.iso8601(3));"
}
}
{% endif %}
output {
lumberjack {
codec => json
hosts => {{ FAILOVER_LOGSTASH_NODES }}
ssl_certificate => "/usr/share/filebeat/ca.crt"
port => 5056
port => 5056
id => "fleet-lumberjack-{{ GLOBALS.hostname }}"
}
}
}
salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
+8 -1
View File
@@ -1,10 +1,17 @@
{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
{%- if grains.role in ['so-heavynode', 'so-receiver'] %}
{%- set HOST = GLOBALS.hostname %}
{%- else %}
{%- set HOST = GLOBALS.manager %}
{%- endif %}
{%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
filter {
ruby {
code => "event.set('[_tmp][logstash_to_redis]', Time.now().utc.iso8601(3));"
}
}
{% endif %}
output {
redis {
host => '{{ HOST }}'
salt/logstash/soc_logstash.yaml
+5
View File
@@ -86,3 +86,8 @@ logstash:
multiline: True
advanced: True
forcedType: "[]string"
latency_metrics:
description: Enable latency metrics within events processed by logstash. Useful for pinpointing log ingest delay.
forcedType: bool
global: False
advanced: True