Move In Day

salt/logstash/dmz_nodes.yaml

@@ -1,3 +1,9 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 # Do not edit this file. Copy it to /opt/so/saltstack/local/salt/logstash/ and make changes there. It should be formatted as a list.
 # logstash:
 # dmz_nodes:
@@ -6,4 +12,4 @@
 #   - mydmznodehostname3
 
 logstash:
-  dmz_nodes:
+  dmz_nodes:

salt/logstash/init.sls

@@ -1,48 +1,30 @@
-# Copyright 2014-2022 Security Onion Solutions, LLC
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
 
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
-  {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
-  {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
-  {% set MANAGER = salt['grains.get']('master') %}
-  {% set MANAGERIP = salt['pillar.get']('global:managerip') %}
+  {% from 'logstash/map.jinja' import REDIS_NODES with context %}
+  {% from 'vars/globals.map.jinja' import GLOBALS %}
 
   # Logstash Section - Decide which pillar to use
-  {% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %}
-  {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %}
-    {% set freq = salt['pillar.get']('manager:freq', '0') %}
-    {% set dstats = salt['pillar.get']('manager:domainstats', '0') %}
-    {% set nodetype = salt['grains.get']('role', '')  %}
-  {% elif grains['role'] == 'so-helix' %}
-    {% set freq = salt['pillar.get']('manager:freq', '0') %}
-    {% set dstats = salt['pillar.get']('manager:domainstats', '0') %}
-    {% set nodetype = salt['grains.get']('role', '')  %}
+  {% set lsheap = salt['pillar.get']('logstash_settings:lsheap') %}
+  {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %}
+    {% set nodetype = GLOBALS.role  %}
   {% endif %}
 
   {% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %}
   {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
   {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 
-  {% from 'logstash/map.jinja' import REDIS_NODES with context %}
-
 include:
   - ssl
-{% if grains.role not in ['so-receiver'] %}
+  {% if GLOBALS.role not in ['so-receiver'] %}
   - elasticsearch
-{% endif %}
+  {% endif %}
 
 # Create the logstash group
 logstashgroup:
@@ -150,7 +132,7 @@ lslogdir:
 
 so-logstash:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}
+    - image: {{ GLOBALS.manager }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash
@@ -171,24 +153,22 @@ so-logstash:
       - /opt/so/log/logstash:/var/log/logstash:rw
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
       - /opt/so/conf/logstash/etc/certs:/usr/share/logstash/certs:ro
-  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
+  {% if GLOBALS.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
   {% endif %}
-  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
+  {% if GLOBALS.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
   {% else %}
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
   {% endif %}
-  {% if grains.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-node'] %}
+  {% if GLOBALS.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode'] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
   {% endif %}
-  {%- if grains['role'] == 'so-eval' %}
+  {%- if GLOBALS.role == 'so-eval' %}
       - /nsm/zeek:/nsm/zeek:ro
       - /nsm/suricata:/suricata:ro
-      - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro
-      - /nsm/wazuh/logs/archives:/wazuh/archives:ro
       - /opt/so/log/fleet/:/osquery/logs:ro
       - /opt/so/log/strelka:/strelka:ro
   {%- endif %}

salt/logstash/map.jinja

@@ -2,7 +2,7 @@
 {% set REDIS_NODES = [] %}
 {% set mainint = salt['pillar.get']('host:mainint') %}
 {% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
-{% if role in ['so-node', 'so-standalone', 'so-managersearch'] %}
+{% if role in ['so-searchnode', 'so-standalone', 'so-managersearch'] %}
   {% set node_data = salt['pillar.get']('logstash:nodes') %}
   {% for node_type, node_details in node_data.items() | sort %}
     {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}

salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja

@@ -1,19 +0,0 @@
-{%- set MANAGER = salt['grains.get']('master') %}
-{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
-{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
-
-input {
-	redis {
-		host => '{{ MANAGER }}'
-		port => 6379
-		data_type => 'pattern_channel'
-		key => 'results_*'
-		type => 'live_query'
-        add_field => {
-        "module" => "osquery"
-          "dataset" => "live_query"
-          }        
-		threads => {{ THREADS }}
-		batch_count => {{ BATCH }}
-	}
-}

salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf

@@ -0,0 +1,12 @@
+input {
+  elastic_agent {
+    port => 5055
+    tags => [ "elastic-agent" ]
+    ssl => true
+    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
+    ssl_certificate => "/usr/share/logstash/filebeat.crt"
+    ssl_key => "/usr/share/logstash/filebeat.key"
+    ssl_verify_mode => "force_peer"
+    ecs_compatibility => v8
+  }
+}

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-zeek"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-import"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja

@@ -1,18 +1,12 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
   if [event_type] == "sflow" {
     elasticsearch {
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-flow"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja

@@ -1,18 +1,12 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
   if [event_type] == "ids" and "import" not in [tags] {
     elasticsearch {
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-ids"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}"
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-syslog"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -11,10 +7,8 @@ output {
       id => "filebeat_modules_metadata_pipeline"
       pipeline => "%{[metadata][pipeline]}"
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-%{[event][module]}-%{+YYYY.MM.dd}"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-osquery"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja

@@ -1,9 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
@@ -32,10 +27,8 @@ output {
     elasticsearch {
       pipeline => "osquery.live_query" 
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-osquery"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -1,18 +1,12 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
   if [dataset] =~ "firewall" {
     elasticsearch {
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-firewall"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-ids"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -11,10 +7,8 @@ output {
       elasticsearch {
         pipeline => "beats.common" 
         hosts => "{{ ES }}"
-  {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
         user => "{{ ES_USER }}"
         password => "{{ ES_PASS }}"
-  {% endif %}
         index => "so-beats"
         ssl => true
         ssl_certificate_verification => false
@@ -24,10 +18,8 @@ output {
       elasticsearch {
         pipeline => "beats.common" 
         hosts => "{{ ES }}"
-  {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
         user => "{{ ES_USER }}"
         password => "{{ ES_PASS }}"
-  {% endif %}
         index => "so-beats"
         ssl => true
         ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}"
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-ossec"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-strelka"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
@@ -12,10 +8,8 @@ output {
       id => "logscan_pipeline"
       pipeline => "logscan.alert"
       hosts => "{{ ES }}"
-    {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-    {% endif %}
       index => "so-logscan"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
@@ -10,10 +6,8 @@ output {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-{% endif %}
       index => "so-rita"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9802_output_kratos.conf.jinja

@@ -1,22 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
-{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
-output {
-  if [module] =~ "kratos" and "import" not in [tags] {
-    elasticsearch {
-      pipeline => "kratos"
-      hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-{% endif %}
-      index => "so-kratos"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -0,0 +1,17 @@
+{%- set ES = salt['grains.get']('master') -%}
+{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
+{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+output {
+  if "elastic-agent" in [tags] and "import" not in [tags] {
+      elasticsearch {
+        hosts => "{{ ES }}"
+        ecs_compatibility => v8
+        data_stream => true
+        user => "{{ ES_USER }}"
+        password => "{{ ES_PASS }}"
+        ssl => true
+        ssl_certificate_verification => false
+      }
+  }
+}
+

salt/logstash/pipelines/config/so/9900_output_endgame.conf.jinja

@@ -1,8 +1,4 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
+{%- set ES = salt['grains.get']('master') -%}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 filter {
@@ -17,10 +13,8 @@ output {
     elasticsearch {
       id => "endgame_es_output"
       hosts => "{{ ES }}"
-    {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
       user => "{{ ES_USER }}"
       password => "{{ ES_PASS }}"
-    {% endif %}
       index => "endgame-%{+YYYY.MM.dd}"
       ssl => true
       ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9997_output_helix.conf.jinja

@@ -1,160 +0,0 @@
-{% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
-{% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
-{% set CBNAME = grains.host %}
-
-filter {
-  if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509|suricata$/ {
-    grok {
-      match => [
-        "source_ip", "^%{IPV4:srcipv4}$",
-        "source_ip",  "(?<srcipv6>^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)"
-        ]
-    }
-    grok {
-      match => [
-        "destination_ip",  "(?<dstipv6>^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)",
-        "destination_ip", "^%{IPV4:dstipv4}$"
-       ]
-    }
-
-    #geoip {
-    #  source => "[source_ip]"
-    #  target => "source_geo"
-    #}
-    #geoip {
-    #  source => "[destination_ip]"
-    #  target => "destination_geo"
-    #}
-    mutate {
-      rename => { "[beat_host][name]" => "sensor" }
-      copy => { "sensor" => "rawmsghostname" }
-      rename => { "message" => "rawmsg" }
-      copy => { "type" => "class" }
-      copy => { "class" => "program"}
-      rename => { "source_port" => "srcport" }
-      rename => { "destination_port" => "dstport" }
-      rename => { "[log][file][path]" => "filepath" }
-      add_field => { "meta_cbid" => "{{ UNIQUEID }}" }
-      add_field => { "meta_cbname" => "{{ CBNAME }}" }
-      remove_field => ["source_ip", "destination_ip", "syslog-host_from"]
-      remove_field => ["beat_host", "timestamp", "type", "log", "@version", "@timestamp"]
-      remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"]
-      remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"]
-    }
-    if "bro_conn" in [class] {
-      mutate {
-        #add_field => { "metaclass" => "connection" }
-        rename => { "original_bytes" => "sentbytes" }
-        rename => { "respond_bytes" => "rcvdbytes" }
-        rename => { "connection_state" => "connstate" }
-        rename => { "uid" => "connectionid" }
-        rename => { "respond_packets" => "rcvdpackets" }
-        rename => { "original_packets" => "sentpackets" }
-        rename => { "respond_ip_bytes" => "rcvdipbytes" }
-        rename => { "original_ip_bytes" => "sentipbytes" }
-        rename => { "local_respond" => "local_resp" }
-        rename => { "local_orig" => "localorig" }
-        rename => { "missed_bytes" => "missingbytes" }
-        rename => { "connection_state_description" => "description" }
-      }
-    }
-    if "bro_dns" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "answers" => "answer" }
-        rename => { "query" => "domain" }
-        rename => { "query_class" => "queryclass" }
-        rename => { "query_class_name" => "queryclassname" }
-        rename => { "query_type" => "querytype" }
-        rename => { "query_type_name" => "querytypename" }
-        rename => { "ra" => "recursionavailable" }
-        rename => { "rd" => "recursiondesired" }
-        rename => { "uid" => "connectionid" }
-        rename => { "ttls" => "ttl" }
-        rename => { "transaction_id" => "transactionid" }
-      }
-    }
-    if "bro_dhcp" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dhcp"}
-        rename => { "message_types" => "direction" }
-        rename => { "uid" => "connectionid" }
-	rename => { "lease_time" => "duration" }
-      }
-    }
-    if "bro_files" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "missing_bytes" => "missingbytes" }
-        rename => { "seen_bytes" => "seenbytes" }
-        rename => { "overflow_bytes" => "overflowbytes" }
-        rename => { "fuid" => "fileid" }
-        rename => { "conn_uids" => "connectionid" }
-        rename => { "is_orig" => "isorig" }
-        rename => { "timed_out" => "timedout" }
-        rename => { "local_orig" => "localorig" }
-        rename => { "file_ip" => "tx_host" }
-      }
-    }
-    if "bro_http" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "virtual_host" => "hostname" }
-        rename => { "status_code" => "statuscode" }
-        rename => { "status_message" => "statusmsg" }
-        rename => { "resp_mime_types" => "rcvdmimetype" }
-        rename => { "resp_fuids" => "rcvdfileid" }
-        rename => { "response_body_len" => "rcvdbodybytes" }
-        rename => { "request_body_len" => "sentbodybytes" }
-        rename => { "uid" => "connectionid" }
-        rename => { "ts"=> "eventtime" }
-        rename => { "@timestamp"=> "eventtime" }
-        rename => { "trans_depth" => "depth" }
-        rename => { "request_body_length" => "sentbodybytes" }
-        rename => { "response_body_length" => "rcvdbodybytes" }
-      }
-    }
-    if "bro_ssl" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "status_code" => "statuscode" }
-        rename => { "status_message" => "statusmsg" }
-        rename => { "resp_mime_types" => "rcvdmimetype" }
-        rename => { "resp_fuids" => "rcvdfileid" }
-        rename => { "response_body_len" => "rcvdbodybytes" }
-        rename => { "request_body_len" => "sentbodybytes" }
-        rename => { "uid" => "connectionid" }
-      }
-    }
-    if "bro_weird" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "name" => "eventname" }
-      }
-    }
-    if "bro_x509" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-	rename => { "certificate_common_name" => "certname" }
-        rename => { "certificate_subject" => "certsubject" }
-	rename => { "issuer_common_name" => "issuer" }
-	rename => { "certificate_issuer" => "issuersubject" }
-	rename => { "certificate_not_valid_before" => "issuetime" }
-	rename => { "certificate_key_type" => "cert_type" }
-      }
-    }
-  }
-}
-
-output {
-  if [class] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509|suricata$/ {
-    http {
-      url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload"
-      http_method => post
-      http_compression => true
-      socket_timeout => 60
-      headers => ["Authorization","{{ HELIX_API_KEY }}"]
-      format => json_batch
-    }
-  }
-}

salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja

@@ -1,25 +0,0 @@
-{%- set MANAGER = salt['grains.get']('master') %}
-{%- set access_key = salt['pillar.get']('minio:access_key', '') %}
-{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %}
-{%- set SIZE_FILE = salt['pillar.get']('s3_settings:size_file', 2048) %}
-{%- set TIME_FILE = salt['pillar.get']('s3_settings:time_file', 1) %}
-{%- set UPLOAD_QUEUE_SIZE = salt['pillar.get']('s3_settings:upload_queue_size', 4) %}
-{%- set ENCODING = salt['pillar.get']('s3_settings:encoding', 'gzip') %}
-output {
-	s3 {
-		access_key_id => "{{ access_key }}"
-        secret_access_key => "{{ access_secret}}"
-        endpoint => "https://{{ MANAGER }}:9595"
-        bucket => "logstash"
-        size_file => {{ SIZE_FILE }}
-        time_file => {{ TIME_FILE }}
-        codec => json
-        encoding => {{ ENCODING }}
-        upload_queue_size => {{ UPLOAD_QUEUE_SIZE }}
-        temporary_directory => "/usr/share/logstash/data/tmp"
-        validate_credentials_on_root_bucket => false
-        additional_settings => {
-            "force_path_style" => true
-            }
-	}
-}