Merge pull request #13877 from Security-Onion-Solutions/2.4/lookuprev2

Initial support for local lookup

salt/elasticfleet/config.sls

@@ -63,6 +63,14 @@ eastatedir:
     - group: 939
     - makedirs: True
 
+custommappingsdir:
+  file.directory:
+    - name: /nsm/custom-mappings
+    - user: 947
+    - group: 939
+    - makedirs: True
+
+
 eapackageupgrade:
   file.managed:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -0,0 +1,35 @@
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "so-ip-mappings",
+  "namespace": "so",
+  "description": "IP Description mappings",
+  "policy_id": "so-grid-nodes_general",
+  "vars": {},
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/custom-mappings/ip-descriptions.csv"
+            ],
+            "data_stream.dataset": "hostnamemappings",
+            "tags": [
+              "so-ip-mappings"
+            ],
+            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
+            "custom": ""
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+
+

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -1,18 +1,45 @@
 output {
-  if "elastic-agent" in [tags] {
-    if [metadata][pipeline] {
-      if [metadata][_id] {
-        elasticsearch {
-          hosts => "{{ GLOBALS.hostname }}"
-          ecs_compatibility => v8
-          data_stream => true
-          user => "{{ ES_USER }}"
-          password => "{{ ES_PASS }}"
-          document_id => "%{[metadata][_id]}"
-          pipeline => "%{[metadata][pipeline]}"
-          silence_errors_in_log => ["version_conflict_engine_exception"]
-          ssl => true
-          ssl_certificate_verification => false
+  if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] {
+    elasticsearch {
+      hosts => "{{ GLOBALS.hostname }}"
+      data_stream => false
+      user => "{{ ES_USER }}"
+      password => "{{ ES_PASS }}"
+      document_id => "%{[metadata][_id]}"
+      index => "so-ip-mappings"
+      silence_errors_in_log => ["version_conflict_engine_exception"]
+      ssl => true
+      ssl_certificate_verification => false
+    }
+  }
+  else {
+    if "elastic-agent" in [tags] {
+      if [metadata][pipeline] {
+        if [metadata][_id] {
+          elasticsearch {
+            hosts => "{{ GLOBALS.hostname }}"
+            ecs_compatibility => v8
+            data_stream => true
+            user => "{{ ES_USER }}"
+            password => "{{ ES_PASS }}"
+            document_id => "%{[metadata][_id]}"
+            pipeline => "%{[metadata][pipeline]}"
+            silence_errors_in_log => ["version_conflict_engine_exception"]
+            ssl => true
+            ssl_certificate_verification => false
+          }
+        }
+        else {
+          elasticsearch {
+            hosts => "{{ GLOBALS.hostname }}"
+            ecs_compatibility => v8
+            data_stream => true
+            user => "{{ ES_USER }}"
+            password => "{{ ES_PASS }}"
+            pipeline => "%{[metadata][pipeline]}"
+            ssl => true
+            ssl_certificate_verification => false
+          }
         }
       }
       else {
@@ -22,22 +49,10 @@ output {
           data_stream => true
           user => "{{ ES_USER }}"
           password => "{{ ES_PASS }}"
-          pipeline => "%{[metadata][pipeline]}"
           ssl => true
           ssl_certificate_verification => false
         }
       }
     }
-    else {
-      elasticsearch {
-        hosts => "{{ GLOBALS.hostname }}"
-        ecs_compatibility => v8
-        data_stream => true
-        user => "{{ ES_USER }}"
-        password => "{{ ES_PASS }}"
-        ssl => true
-        ssl_certificate_verification => false
-      }
-    }
   }
 }