Merge pull request #9650 from Security-Onion-Solutions/fix/elastic_agent_add_import_mode

Elastic Agent - Import Mode

salt/allowed_states.map.jinja

@@ -107,7 +107,8 @@
           'zeek',
           'schedule',
           'tcpreplay',
-          'docker_clean'
+          'docker_clean',
+          'elastic-fleet'
           ],
       'so-manager': [
           'salt.master',

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -66,10 +66,10 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS
 echo
 
 # RITA Logs
-echo
+#echo
-echo "Setting up RITA package policy..."
+#echo "Setting up RITA package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita\n- if:\n    log.file.path: beacons.csv\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.beacon\n- if:\n    regexp:\n      log.file.path: \"*connections.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.connection\n- if:\n    log.file.path: \"exploded-dns.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.dns" }}}}}}'
+#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita\n- if:\n    log.file.path: beacons.csv\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.beacon\n- if:\n    regexp:\n      log.file.path: \"*connections.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.connection\n- if:\n    log.file.path: \"exploded-dns.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.dns" }}}}}}'
-echo
+#echo
 
 # Elasticsearch logs
 echo

salt/common/tools/sbin/so-elastic-fleet-setup

@@ -19,10 +19,22 @@ printf "\n"
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}'
 printf "\n\n"
 
-# Create Logstash Output payload
+# Configure certificates
 mkdir -p /opt/so/conf/elastic-fleet/certs
 cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs
 cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs
+
+{% if grains.role == 'so-import' %}
+# Add SO-Manager Elasticsearch Ouput
+ESCACRT=$(openssl x509 -in  /opt/so/conf/elastic-fleet/certs/intca.crt)
+JSON_STRING=$( jq -n \
+                  --arg ESCACRT "$ESCACRT" \
+                  '{"name":"so-manager_elasticsearch2","id":"so-manager_elasticsearch2","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' )
+curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+printf "\n\n"
+
+{% else %}
+# Create Logstash Output payload
 LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt)
 LOGSTASHKEY=$(openssl rsa -in  /opt/so/conf/elastic-fleet/certs/elasticfleet.key)
 LOGSTASHCA=$(openssl x509 -in  /opt/so/conf/elastic-fleet/certs/intca.crt)
@@ -30,12 +42,13 @@ JSON_STRING=$( jq -n \
                   --arg LOGSTASHCRT "$LOGSTASHCRT" \
                   --arg LOGSTASHKEY "$LOGSTASHKEY" \
                   --arg LOGSTASHCA "$LOGSTASHCA" \
-                  '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}'                
+                  '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}'
                   )
 
 # Add SO-Manager Logstash Ouput
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 printf "\n\n"
+{%- endif %}
 
 # Add Elastic Fleet Integrations
 

salt/common/tools/sbin/so-image-common

@@ -36,7 +36,9 @@ container_list() {
       "so-steno"
       "so-suricata"
       "so-telegraf"
-      "so-zeek" 
+      "so-zeek"
+      "so-elastic-agent"
+      "so-elastic-agent-builder"
     )
   elif [ $MANAGERCHECK != 'so-helix' ]; then
     TRUSTED_CONTAINERS=(

salt/elasticsearch/defaults.yaml

@@ -2677,6 +2677,9 @@ elasticsearch:
       delete: 365
       index_sorting: False
       index_template:
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
         index_patterns:
           - logs-kratos-so*
         template:

salt/firewall/assigned_hostgroups.map.yaml

@@ -436,6 +436,7 @@ role:
               - {{ portgroups.influxdb }}
               - {{ portgroups.elasticsearch_rest }}
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.elastic_agent_control }}
           sensors:
             portgroups:
               - {{ portgroups.beats_5044 }}
@@ -453,6 +454,10 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+          elastic_agent_endpoint:
+            portgroups:
+              - {{ portgroups.elastic_agent_control }}
+              - {{ portgroups.elastic_agent_data }}
           analyst:
             portgroups:
               - {{ portgroups.nginx }}

salt/top.sls

@@ -344,6 +344,7 @@ base:
     - zeek
     - schedule
     - docker_clean
+    - elastic-fleet
 
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound

setup/so-setup

@@ -603,9 +603,9 @@ if ! [[ -f $install_opt_file ]]; then
 		add_web_user
 		info "Restarting SOC to pick up initial user"
 		logCmd "so-soc-restart"
+		title "Setting up Elastic Fleet"
+                logCmd "so-elastic-fleet-setup"
 		if [[ ! $is_import ]]; then
-			title "Setting up Elastic Fleet"
-			logCmd "so-elastic-fleet-setup"
 			title "Setting up Playbook"
 			logCmd "so-playbook-reset"
 		fi