From dc07adca635f51ac3ac7be3bf422aebf603f02d5 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 1 Mar 2022 15:05:08 +0000 Subject: [PATCH 1/2] Rename ingest.timestamp to event.ingested --- salt/elasticsearch/files/ingest/suricata.common | 2 +- salt/elasticsearch/files/ingest/syslog | 2 +- salt/elasticsearch/files/ingest/zeek.common | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 8e286d2b5..a8be6d46b 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -12,7 +12,7 @@ { "remove":{ "field": "dataset", "ignore_failure": true } }, { "rename":{ "field": "message2.event_type", "target_field": "dataset", "ignore_failure": true } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, - { "set": { "field": "ingest.timestamp", "value": "{{@timestamp}}" } }, + { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, { "pipeline": { "if": "ctx?.dataset != null", "name": "suricata.{{dataset}}" } } diff --git a/salt/elasticsearch/files/ingest/syslog b/salt/elasticsearch/files/ingest/syslog index 91b14d6f4..b19551609 100644 --- a/salt/elasticsearch/files/ingest/syslog +++ b/salt/elasticsearch/files/ingest/syslog @@ -111,7 +111,7 @@ } }, { "set": { - "field": "ingest.timestamp", + "field": "event.ingested", "value": "{{ @timestamp }}" } }, { diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index e7b898c6f..3265e20be 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -1,8 +1,8 @@ { "description" : "zeek.common", "processors" : [ - { "rename": { "if": "ctx.message2?.ts != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, - { "set": { "if": "ctx.message2?.ts == null", "field": "ingest.timestamp", "value": "{{ @timestamp }}" } }, + { "rename": { "if": "ctx.message2?.ts != null", "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, + { "set": { "if": "ctx.message2?.ts == null", "field": "event.ingested", "value": "{{ @timestamp }}" } }, { "rename": { "field": "message2.uid", "target_field": "log.id.uid", "ignore_missing": true } }, { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, From 038dc49098edce19a428b5eca1406a9abb4ebeb2 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 1 Mar 2022 15:06:28 +0000 Subject: [PATCH 2/2] Temporarily increase field limit before trimming efforts --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index b9986d051..87905eddb 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3915,7 +3915,7 @@ elasticsearch: index: mapping: total_fields: - limit: 3000 + limit: 4000 sort: field: "@timestamp" order: desc