Merge pull request #9986 from Security-Onion-Solutions/fix/elastic_agent_template_changes

Elastic Agent template changes

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,8 @@
 elasticsearch:
   config:
-    node: {}
+    node:
+      attr:
+        box_type: hot
     cluster:
       routing:
         allocation:
@@ -55,7 +57,75 @@ elasticsearch:
         elasticsearch:
           deprecation: ERROR
   index_settings:
-    so-logs-elastic_agent.apm_server:
+    so-logs:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-*-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5001
+              sort:
+                field: "@timestamp"
+                order: desc
+          mappings:
+            _meta:
+              package:
+                name: elastic_agent
+              managed_by: security_onion
+              managed: true
+        composed_of:
+          - "so-data-streams-mappings"
+          - "so-logs-mappings"
+          - "so-logs-settings"
+        priority: 225
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true  
+    logs-osquery-manager:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - ".logs-osquery*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+        priority: 501
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    logs-elastic_agent.apm_server:
       index_sorting: False
       index_template:
         index_patterns:
@@ -77,8 +147,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.apm_server@package"
+          - "logs-elastic_agent.apm_server@package"
-          - "so-logs-elastic_agent.apm_server@custom"
+          - "logs-elastic_agent.apm_server@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -109,7 +179,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.auditbeat:
+    logs-elastic_agent.auditbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -131,8 +201,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.auditbeat@package"
+          - "logs-elastic_agent.auditbeat@package"
-          - "so-logs-elastic_agent.auditbeat@custom"
+          - "logs-elastic_agent.auditbeat@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -163,7 +233,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.cloudbeat:
+    logs-elastic_agent.cloudbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -185,8 +255,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.cloudbeat@package"
+          - "logs-elastic_agent.cloudbeat@package"
-          - "so-logs-elastic_agent.cloudbeat@custom"
+          - "logs-elastic_agent.cloudbeat@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -214,7 +284,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.endpoint_security:
+    logs-elastic_agent.endpoint_security:
       index_sorting: False
       index_template:
         index_patterns:
@@ -236,8 +306,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.endpoint_security@package"
+          - "logs-elastic_agent.endpoint_security@package"
-          - "so-logs-elastic_agent.endpoint_security@custom"
+          - "logs-elastic_agent.endpoint_security@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -268,7 +338,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.filebeat:
+    logs-elastic_agent.filebeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -290,11 +360,14 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.filebeat@package"
+          - "logs-elastic_agent.filebeat@package"
-          - "so-logs-elastic_agent.filebeat@custom"
+          - "logs-elastic_agent.filebeat@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
       policy:
         phases:
           hot:
@@ -319,7 +392,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.fleet_server:
+    logs-elastic_agent.fleet_server:
       index_sorting: False
       index_template:
         index_patterns:
@@ -341,8 +414,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.fleet_server@package"
+          - "logs-elastic_agent.fleet_server@package"
-          - "so-logs-elastic_agent.fleet_server@custom"
+          - "logs-elastic_agent.fleet_server@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -373,7 +446,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.heartbeat:
+    logs-elastic_agent.heartbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -395,8 +468,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.heartbeat@package"
+          - "logs-elastic_agent.heartbeat@package"
-          - "so-logs-elastic_agent.heartbeat@custom"
+          - "logs-elastic_agent.heartbeat@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -424,7 +497,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent:
+    logs-elastic_agent:
       index_sorting: False
       index_template:
         index_patterns:
@@ -446,8 +519,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent@package"
+          - "logs-elastic_agent@package"
-          - "so-logs-elastic_agent@custom"
+          - "logs-elastic_agent@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -478,7 +551,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.metricbeat:
+    logs-elastic_agent.metricbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -500,8 +573,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.metricbeat@package"
+          - "logs-elastic_agent.metricbeat@package"
-          - "so-logs-elastic_agent.metricbeat@custom"
+          - "logs-elastic_agent.metricbeat@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -532,7 +605,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.osquerybeat:
+    logs-elastic_agent.osquerybeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -554,8 +627,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.osquerybeat@package"
+          - "logs-elastic_agent.osquerybeat@package"
-          - "so-logs-elastic_agent.osquerybeat@custom"
+          - "logs-elastic_agent.osquerybeat@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200
@@ -586,7 +659,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.packetbeat:
+    logs-elastic_agent.packetbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -608,8 +681,8 @@ elasticsearch:
               managed_by: security_onion
               managed: true
         composed_of:
-          - "so-logs-elastic_agent.packetbeat@package"
+          - "logs-elastic_agent.packetbeat@package"
-          - "so-logs-elastic_agent.packetbeat@custom"
+          - "logs-elastic_agent.packetbeat@custom"
           - "so-fleet_globals-1"
           - "so-fleet_agent_id_verification-1"
         priority: 200

salt/elasticsearch/templates/component/elastic-agent/so-data-streams-mappings.json

@@ -0,0 +1,67 @@
+{
+        "template": {
+          "mappings": {
+            "dynamic_templates": [
+              {
+                "match_ip": {
+                  "mapping": {
+                    "type": "ip"
+                  },
+                  "match_mapping_type": "string",
+                  "match": "ip"
+                }
+              },
+              {
+                "match_message": {
+                  "mapping": {
+                    "type": "match_only_text"
+                  },
+                  "match_mapping_type": "string",
+                  "match": "message"
+                }
+              },
+              {
+                "strings_as_keyword": {
+                  "mapping": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "match_mapping_type": "string"
+                }
+              }
+            ],
+            "date_detection": false,
+            "properties": {
+              "@timestamp": {
+                "type": "date"
+              },
+              "ecs": {
+                "properties": {
+                  "version": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "data_stream": {
+                "properties": {
+                  "namespace": {
+                    "type": "constant_keyword"
+                  },
+                  "dataset": {
+                    "type": "constant_keyword"
+                  }
+                }
+              },
+              "host": {
+                "type": "object"
+              }
+            }
+          }
+        },
+        "version": 2,
+        "_meta": {
+          "managed": true,
+          "description": "general mapping conventions for data streams"
+        }
+      }

salt/elasticsearch/templates/component/elastic-agent/so-logs-mappings.json

@@ -0,0 +1,21 @@
+      {
+        "template": {
+          "mappings": {
+            "properties": {
+              "data_stream": {
+                "properties": {
+                  "type": {
+                    "type": "constant_keyword",
+                    "value": "logs"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "version": 2,
+        "_meta": {
+          "managed": true,
+          "description": "default mappings for the logs index template installed by x-pack"
+        }
+      }

salt/elasticsearch/templates/component/elastic-agent/so-logs-settings.json

@@ -0,0 +1,22 @@
+{
+        "template": {
+          "settings": {
+            "index": {
+              "lifecycle": {
+                "name": "logs"
+              },
+              "codec": "best_compression",
+              "query": {
+                "default_field": [
+                  "message"
+                ]
+              }
+            }
+          }
+        },
+        "version": 2,
+        "_meta": {
+          "managed": true,
+          "description": "default settings for the logs index template installed by x-pack"
+        }
+      }