securityonion/salt/elasticsearch/files/ingest-dynamic/common

{%- set HIGHLANDER = salt['pillar.get']('global:highlander', False) -%}
{%- raw -%}
{
  "description" : "common",
  "processors" : [
    { 	
	"geoip":  { 
		"field": "destination.ip",
		"target_field": "destination.geo",
		"database_file": "GeoLite2-City.mmdb",
		"ignore_missing": true,
		"ignore_failure": true,
		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
	}
    },
    { 
	"geoip":  {
		"field": "source.ip",
		"target_field": "source.geo",
		"database_file": "GeoLite2-City.mmdb",
		"ignore_missing": true,
		"ignore_failure": true,
		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
	}
    },
    {
        "geoip":  {
                "field": "destination.ip",
                "target_field": "destination.as",
                "database_file": "GeoLite2-ASN.mmdb",
                "ignore_missing": true,
                "ignore_failure": true,
                "properties": ["ip", "asn", "organization_name", "network"]
        }
    },
    {
        "geoip":  {
                "field": "source.ip",
                "target_field": "source.as",
                "database_file": "GeoLite2-ASN.mmdb",
                "ignore_missing": true,
                "ignore_failure": true,
                "properties": ["ip", "asn", "organization_name", "network"]
        }
    },
    { "rename":          { "field": "destination.as.organization_name", "target_field": "destination.as.organization.name", "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "source.as.organization_name", "target_field": "source.as.organization.name", "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "destination.as.asn", "target_field": "destination.as.number", "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "source.as.asn", "target_field": "source.as.number", "ignore_failure": true, "ignore_missing": true  } },
    { "set":             { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
    { "set":             { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
    { "set":             { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },
    { "set":             { "if": "ctx.event?.severity == 4",   "field": "event.severity_label", "value": "critical", "override": true }  },
    { "rename":          { "field": "fields.category",              "target_field": "event.category",                  "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "dataset",             "target_field": "event.dataset",                "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
    { "rename":          { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
    { "lowercase":       { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },
    { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
    { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
    { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
    { "set":             { "field": "event.dataset", "ignore_empty_value":true, "copy_from": "event.dataset_temp"   } },
    { "set":             { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } },
    { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
    { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
    { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
{%- endraw %}
{%- if HIGHLANDER %}
    ,
    {
      "pipeline": {
        "name": "ecs"
      }
    }
{%- endif %}
{%- raw %}
    ,
    { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }
  ]
}
{% endraw %}