Merge remote-tracking branch 'remotes/origin/2.4/dev' into 2.4/firewall

salt/allowed_states.map.jinja

@@ -107,7 +107,8 @@
           'zeek',
           'schedule',
           'tcpreplay',
-          'docker_clean'
+          'docker_clean',
+          'elastic-fleet'
           ],
       'so-manager': [
           'salt.master',

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -66,10 +66,10 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS
 echo
 
 # RITA Logs
-echo
-echo "Setting up RITA package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita\n- if:\n    log.file.path: beacons.csv\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.beacon\n- if:\n    regexp:\n      log.file.path: \"*connections.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.connection\n- if:\n    log.file.path: \"exploded-dns.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.dns" }}}}}}'
-echo
+#echo
+#echo "Setting up RITA package policy..."
+#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita\n- if:\n    log.file.path: beacons.csv\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.beacon\n- if:\n    regexp:\n      log.file.path: \"*connections.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.connection\n- if:\n    log.file.path: \"exploded-dns.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.dns" }}}}}}'
+#echo
 
 # Elasticsearch logs
 echo

salt/common/tools/sbin/so-elastic-fleet-setup

@@ -19,10 +19,22 @@ printf "\n"
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}'
 printf "\n\n"
 
-# Create Logstash Output payload
+# Configure certificates
 mkdir -p /opt/so/conf/elastic-fleet/certs
 cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs
 cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs
+
+{% if grains.role == 'so-import' %}
+# Add SO-Manager Elasticsearch Ouput
+ESCACRT=$(openssl x509 -in  /opt/so/conf/elastic-fleet/certs/intca.crt)
+JSON_STRING=$( jq -n \
+                  --arg ESCACRT "$ESCACRT" \
+                  '{"name":"so-manager_elasticsearch2","id":"so-manager_elasticsearch2","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' )
+curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+printf "\n\n"
+
+{% else %}
+# Create Logstash Output payload
 LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt)
 LOGSTASHKEY=$(openssl rsa -in  /opt/so/conf/elastic-fleet/certs/elasticfleet.key)
 LOGSTASHCA=$(openssl x509 -in  /opt/so/conf/elastic-fleet/certs/intca.crt)
@@ -30,12 +42,13 @@ JSON_STRING=$( jq -n \
                   --arg LOGSTASHCRT "$LOGSTASHCRT" \
                   --arg LOGSTASHKEY "$LOGSTASHKEY" \
                   --arg LOGSTASHCA "$LOGSTASHCA" \
-                  '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}'                
+                  '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}'
                   )
 
 # Add SO-Manager Logstash Ouput
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 printf "\n\n"
+{%- endif %}
 
 # Add Elastic Fleet Integrations
 

salt/common/tools/sbin/so-image-common

@@ -36,7 +36,9 @@ container_list() {
       "so-steno"
       "so-suricata"
       "so-telegraf"
-      "so-zeek" 
+      "so-zeek"
+      "so-elastic-agent"
+      "so-elastic-agent-builder"
     )
   elif [ $MANAGERCHECK != 'so-helix' ]; then
     TRUSTED_CONTAINERS=(

salt/elasticsearch/defaults.yaml

@@ -2677,6 +2677,9 @@ elasticsearch:
       delete: 365
       index_sorting: False
       index_template:
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
         index_patterns:
           - logs-kratos-so*
         template:

salt/firewall/assigned_hostgroups.map.yaml

@@ -454,6 +454,7 @@ role:
               - {{ portgroups.influxdb }}
               - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.elastic_agent_control }}
           sensors:
             portgroups:
               - {{ portgroups.beats_5044 }}
@@ -471,6 +472,10 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+          elastic_agent_endpoint:
+            portgroups:
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
           analyst:
             portgroups:
               - {{ portgroups.nginx }}

salt/logstash/pipelines/config/so/0009_input_beats.conf

@@ -1,11 +0,0 @@
-input {
-  beats {
-    port => "5044"
-    tags => [ "beat-ext" ]
-  }
-}
-filter {
-  mutate {
-    rename => {"@metadata" => "metadata"}
-  }
-}

salt/logstash/pipelines/config/so/0010_input_hhbeats.conf

@@ -1,40 +0,0 @@
-input {
-  beats {
-    port => "5644"
-    ssl => true
-    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
-    ssl_certificate => "/usr/share/logstash/filebeat.crt"
-    ssl_key => "/usr/share/logstash/filebeat.key"
-    #tags => [ "beat" ]
-  }
-}
-filter {
-  if [type] == "ids" or [type] =~ "bro" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_field => { "sensor_name" => "%{[beat][name]}" }
-      add_field => { "syslog-host_from" => "%{[beat][name]}" }
-      remove_field => [ "beat", "prospector", "input", "offset" ]
-    }
-  }
-  if [type] =~ "ossec" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_field => { "syslog-host_from" => "%{[beat][name]}" }
-      remove_field => [ "beat", "prospector", "input", "offset" ]
-    }
-   }
-    if [type] == "osquery" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_tag => ["osquery"]
-        	}
-   json {
-    source => "message"
-    target => "osquery"
-        }
-  }
-}

salt/logstash/pipelines/config/so/0800_input_eval.conf

@@ -1,204 +0,0 @@
-# Updated by: Mike Reeves
-# Last Update: 11/1/2018
-
-input {
-  file {
-	  path => "/suricata/eve.json"
-		type => "ids"
-                add_field => { "engine" => "suricata" }
-	}
-	file {
-		path => "/nsm/zeek/logs/current/conn*.log"
-		type => "zeek.conn"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dce_rpc*.log"
-		type => "zeek.dce_rpc"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dhcp*.log"
-		type => "zeek.dhcp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dnp3*.log"
-		type => "zeek.dnp3"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dns*.log"
-		type => "zeek.dns"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/dpd*.log"
-		type => "zeek.dpd"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/files*.log"
-		type => "zeek.files"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ftp*.log"
-		type => "zeek.ftp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/http*.log"
-		type => "zeek.http"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/intel*.log"
-		type => "zeek.intel"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/irc*.log"
-		type => "zeek.irc"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/kerberos*.log"
-		type => "zeek.kerberos"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/modbus*.log"
-		type => "zeek.modbus"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/mysql*.log"
-		type => "zeek.mysql"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/notice*.log"
-		type => "zeek.notice"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ntlm*.log"
-		type => "zeek.ntlm"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/pe*.log"
-		type => "zeek.pe"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/radius*.log"
-		type => "zeek.radius"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/rdp*.log"
-		type => "zeek.rdp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/rfb*.log"
-		type => "zeek.rfb"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/signatures*.log"
-		type => "zeek.signatures"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/sip*.log"
-		type => "zeek.sip"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/smb_files*.log"
-		type => "zeek.smb_files"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/smb_mapping*.log"
-		type => "zeek.smb_mapping"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/smtp*.log"
-		type => "zeek.smtp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/snmp*.log"
-		type => "zeek.snmp"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/socks*.log"
-		type => "zeek.socks"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/software*.log"
-		type => "zeek.software"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ssh*.log"
-		type => "zeek.ssh"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/ssl*.log"
-		type => "zeek.ssl"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/syslog*.log"
-		type => "zeek.syslog"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/tunnel*.log"
-		type => "zeek.tunnels"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/weird*.log"
-		type => "zeek.weird"
-	    	tags => ["zeek"]
-	}
-	file {
-		path => "/nsm/zeek/logs/current/x509*.log"
-		type => "zeek.x509"
-	    	tags => ["zeek"]
-	}
-  file {
-    path => "/wazuh/alerts/alerts.json"
-    type => "ossec"
-  }
-#  file {
-#    path => "/wazuh/archives/archives.json"
-#    type => "ossec_archive"
-#  }
-  file {
-    path => "/osquery/logs/result.log"
-    type => "osquery"
-    tags => ["osquery"]
-  }
-  file {
-    path => "/strelka/strelka.log"
-    type => "strelka"
-  }
-}
-filter {
-  if "import" in [tags] {
-	mutate {
-	  #add_tag => [ "conf_file_0007"]
-	}
-  }
-}

salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja

@@ -1,23 +0,0 @@
-{%- if grains.role == 'so-heavynode' %}
-{%- set HOST = GLOBALS.hostname %}
-{%- else %}
-{%- set HOST = GLOBALS.manager %}
-{% endif -%}
-{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
-{%- set access_key = salt['pillar.get']('minio:access_key', '') %}
-{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %}
-{%- set INTERVAL = salt['pillar.get']('s3_settings:interval', 5) %}
-input {
-	s3 {
-        access_key_id => "{{ access_key }}"
-        secret_access_key => "{{ access_secret }}"
-        endpoint => "https://{{ HOST }}:9595"
-        bucket => "logstash"
-        delete => true
-        interval => {{ INTERVAL }}
-        codec => json
-        additional_settings => {
-            "force_path_style" => true
-            }
-    }
-}

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "zeek" and "import" not in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-zeek"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if "import" in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-import"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "syslog" {
-    elasticsearch {
-      pipeline => "%{module}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-syslog"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja

@@ -1,14 +0,0 @@
-output {
-  if "filebeat" in [metadata][pipeline] {
-    elasticsearch {
-      id => "filebeat_modules_metadata_pipeline"
-      pipeline => "%{[metadata][pipeline]}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-%{[event][module]}-%{+YYYY.MM.dd}"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "osquery" and "live_query" not in [dataset] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}" 
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-osquery"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -1,12 +0,0 @@
-output {
-  if [dataset] =~ "firewall" {
-    elasticsearch {
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-firewall"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "suricata" and "import" not in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-ids"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -1,26 +0,0 @@
-output {
-  if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] {
-    if [metadata][_id] {
-      elasticsearch {
-        pipeline => "beats.common" 
-        hosts => "{{ GLOBALS.manager }}"
-        user => "{{ ES_USER }}"
-        password => "{{ ES_PASS }}"
-        index => "so-beats"
-        ssl => true
-        ssl_certificate_verification => false
-        document_id => "%{[metadata][_id]}" 
-      }
-      } else {
-      elasticsearch {
-        pipeline => "beats.common" 
-        hosts => "{{ GLOBALS.manager }}"
-        user => "{{ ES_USER }}"
-        password => "{{ ES_PASS }}"
-        index => "so-beats"
-        ssl => true
-        ssl_certificate_verification => false
-      }
-  }
- }
-}

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "ossec" {
-    elasticsearch {
-      pipeline => "%{module}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-ossec"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -1,14 +0,0 @@
-output {
-  if [module] =~ "strelka" {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}" 
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-strelka"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}
-

salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja

@@ -1,14 +0,0 @@
-output {
-  if [module] =~ "logscan" {
-    elasticsearch {
-      id => "logscan_pipeline"
-      pipeline => "logscan.alert"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-logscan"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja

@@ -1,13 +0,0 @@
-output {
-  if [module] =~ "rita" and "import" not in [tags] {
-    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
-      hosts => "{{ GLOBALS.manager }}"
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-      index => "so-rita"
-      ssl => true
-      ssl_certificate_verification => false
-    }
-  }
-}

salt/top.sls

@@ -344,6 +344,7 @@ base:
     - zeek
     - schedule
     - docker_clean
+    - elastic-fleet
 
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound