CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-05-26 04:55:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

add ingest latency metrics

Browse Source
This commit is contained in:
reyesj2
2026-05-11 16:51:38 -05:00
parent 49a643fff4
commit 492ae80da7
11 changed files with 162 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/elasticsearch/files/ingest/global@custom
+75 -2
View File
@@ -177,12 +177,84 @@
"description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
}
},
{
"script": {
"description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
"lang": "painless",
"if": "ctx.event?.ingested != null && ctx.event?.created == null",
"ignore_failure": true,
"source": "ctx.putIfAbsent('_tmp', [:]); ctx._tmp.event_ingested_pre_fleet = ctx.event.ingested;"
}
},
{
"pipeline": {
"name": ".fleet_final_pipeline-1",
"ignore_missing_pipeline": true
}
},
{
"script": {
"description": "Calculate time from Elastic Agent to Logstash.",
"lang": "painless",
"if": "ctx._tmp?.logstash_from_agent != null",
"ignore_failure": true,
"source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_logstash = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_agent));"
}
},
{
"script": {
"description": "Calculate time from Logstash to Redis",
"lang": "painless",
"if": "ctx._tmp?.logstash_from_agent != null && ctx._tmp?.logstash_to_redis != null",
"ignore_failure": true,
"source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_redis = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_agent), ZonedDateTime.parse(ctx._tmp.logstash_to_redis));"
}
},
{
"script": {
"description": "Calculate time message spends in redis queue (logstash delay in pulling event).",
"lang": "painless",
"if": "ctx._tmp?.logstash_to_redis != null && ctx._tmp?.logstash_from_redis != null",
"ignore_failure": true,
"source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_redis_to_logstash = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_to_redis), ZonedDateTime.parse(ctx._tmp.logstash_from_redis));"
}
},
{
"script": {
"description": "Calculate time from Logstash to Elasticsearch (after read from Redis).",
"lang": "painless",
"if": "ctx._tmp?.logstash_from_redis != null",
"ignore_failure": true,
"source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_redis), metadata().now);"
}
},
{
"script": {
"description": "Calculate time from Elastic Agent to Kafka.",
"lang": "painless",
"if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
"ignore_failure": true,
"source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_kafka = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
}
},
{
"script": {
"description": "Calculate time message spends in Kafka queue (logstash delay in pulling event).",
"lang": "painless",
"if": "ctx._tmp?.logstash_from_kafka != null && ctx.metadata?.kafka?.timestamp != null && ctx._tmp?.logstash_from_agent == null",
"ignore_failure": true,
"source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_queue = ChronoUnit.SECONDS.between(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(ctx.metadata.kafka.timestamp.toString())), ZoneId.of('UTC')), ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
}
},
{
"script": {
"description": "Calculate time from Logstash to Elasticsearch (after read from Kafka).",
"lang": "painless",
"if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
"ignore_failure": true,
"source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_kafka), metadata().now);"
}
},
{
"remove": {
"field": "event.agent_id_status",
@@ -202,11 +274,12 @@
"event.dataset_temp",
"dataset_tag_temp",
"module_temp",
"datastream_dataset_temp"
"datastream_dataset_temp",
"_tmp"
],
"ignore_missing": true,
"ignore_failure": true
}
}
]
}
}