Merge pull request #10469 from Security-Onion-Solutions/fix/elastic_fleet_dynamic_configuration

Use Fleet pillar instead of Zeek

pillar/top.sls

@@ -52,6 +52,8 @@ base:
     - influxdb.adv_influxdb
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
     - elastalert.soc_elastalert
     - elastalert.adv_elastalert
     - backup.soc_backup
@@ -91,6 +93,8 @@ base:
     - kratos.soc_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
     - elastalert.soc_elastalert
     - elastalert.adv_elastalert
     - manager.soc_manager
@@ -149,6 +153,8 @@ base:
     - influxdb.adv_influxdb
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
     - elastalert.soc_elastalert
     - elastalert.adv_elastalert
     - manager.soc_manager
@@ -244,6 +250,8 @@ base:
     - kratos.soc_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
     - elastalert.soc_elastalert
     - elastalert.adv_elastalert
     - manager.soc_manager
@@ -283,6 +291,8 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 

salt/elasticfleet/defaults.yaml

@@ -6,3 +6,18 @@ elasticfleet:
       es_token: ''
       grid_enrollment: ''
       url: ''
+  logging:
+    zeek:
+      excluded:
+        - broker
+        - capture_loss
+        - ecat_arp_info
+        - known_hosts
+        - known_services
+        - loaded_scripts
+        - ntp
+        - packet_filter
+        - reporter
+        - stats
+        - stderr
+        - stdout

salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json

@@ -1,4 +1,4 @@
-{% from 'zeek/config.map.jinja' import ZEEKMERGED %}
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 {%- raw -%}
 {
   "package": {
@@ -22,7 +22,7 @@
             "data_stream.dataset": "import",
             "tags": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ZEEKMERGED.logging.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json

@@ -1,4 +1,4 @@
-{% from 'zeek/config.map.jinja' import ZEEKMERGED %}
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 {%- raw -%}
 {
   "package": {
@@ -23,7 +23,7 @@
             "data_stream.dataset": "zeek",
             "tags": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ZEEKMERGED.logging.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
           }
         }
       }

salt/elasticfleet/soc_elasticfleet.yaml

@@ -3,6 +3,12 @@ elasticfleet:
     description: You can enable or disable Elastic Fleet.
     advanced: True
     helpLink: elastic-fleet.html
+  logging:
+    zeek:
+      excluded:
+        description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors.
+        forcedType: "[]string"
+        helpLink: zeek.html 
   config:
     server:
       endpoints_enrollment:

salt/zeek/soc_zeek.yaml

@@ -2,11 +2,6 @@ zeek:
   enabled:
     description: You can enable or disable ZEEK on all sensors or a single sensor.
     helpLink: zeek.html
-  logging:
-    excluded:
-      description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors.
-      forcedType: "[]string"
-      helpLink: zeek.html
   config:
     local:
       load:

setup/so-functions

@@ -898,6 +898,7 @@ create_local_nids_rules() {
 }
 
 create_manager_pillars() {
+	elasticfleet_pillar
 	elasticsearch_pillar
 	logstash_pillar
 	manager_pillar
@@ -1120,6 +1121,12 @@ docker_seed_registry() {
 	fi
 }
 
+elasticfleet_pillar() {
+    logCmd "mkdir -p $local_salt_dir/pillar/elasticfleet"
+    touch $adv_elasticfleet_pillar_file
+    touch $elasticfleet_pillar_file
+}
+
 elasticsearch_pillar() {
 	title "Create Advanced File"
 	logCmd "touch $adv_elasticsearch_pillar_file"

setup/so-variables

@@ -82,6 +82,12 @@ export global_pillar_file
 adv_global_pillar_file="$local_salt_dir/pillar/global/adv_global.sls"
 export adv_global_pillar_file
 
+elasticfleet_pillar_file="$local_salt_dir/pillar/elasticfleet/soc_elasticfleet.sls"
+export elasticfleet_pillar_file
+
+adv_elasticfleet_pillar_file="$local_salt_dir/pillar/elasticfleet/adv_elasticfleet.sls"
+export adv_elasticfleet_pillar_file
+
 elasticsearch_pillar_file="$local_salt_dir/pillar/elasticsearch/soc_elasticsearch.sls"
 export elasticsearch_pillar_file