diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 2f6cc60a0..823b7b647 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -107,7 +107,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'elastic-fleet' ], 'so-manager': [ 'salt.master', diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index b3148f7aa..f409d9a17 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -66,10 +66,10 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS echo # RITA Logs -echo -echo "Setting up RITA package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' -echo +#echo +#echo "Setting up RITA package policy..." +#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' +#echo # Elasticsearch logs echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 4e3d7639c..c945aeec3 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,10 +19,22 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" -# Create Logstash Output payload +# Configure certificates mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs + +{% if grains.role == 'so-import' %} +# Add SO-Manager Elasticsearch Ouput +ESCACRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) +JSON_STRING=$( jq -n \ + --arg ESCACRT "$ESCACRT" \ + '{"name":"so-manager_elasticsearch2","id":"so-manager_elasticsearch2","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +printf "\n\n" + +{% else %} +# Create Logstash Output payload LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) @@ -30,12 +42,13 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' + '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' ) # Add SO-Manager Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" +{%- endif %} # Add Elastic Fleet Integrations diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 3851d8b4a..945d3f4ed 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -36,7 +36,9 @@ container_list() { "so-steno" "so-suricata" "so-telegraf" - "so-zeek" + "so-zeek" + "so-elastic-agent" + "so-elastic-agent-builder" ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 11b445c62..3d1182255 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -2677,6 +2677,9 @@ elasticsearch: delete: 365 index_sorting: False index_template: + data_stream: + hidden: false + allow_custom_routing: false index_patterns: - logs-kratos-so* template: diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 6e249ec93..0f16228b2 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -436,6 +436,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elastic_agent_control }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -453,6 +454,10 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + elastic_agent_endpoint: + portgroups: + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} analyst: portgroups: - {{ portgroups.nginx }} diff --git a/salt/top.sls b/salt/top.sls index e29d3b081..4b8531f4d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -344,6 +344,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': - match: compound diff --git a/setup/so-setup b/setup/so-setup index e96ef5f1b..9ecbed08c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -603,9 +603,9 @@ if ! [[ -f $install_opt_file ]]; then add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" + title "Setting up Elastic Fleet" + logCmd "so-elastic-fleet-setup" if [[ ! $is_import ]]; then - title "Setting up Elastic Fleet" - logCmd "so-elastic-fleet-setup" title "Setting up Playbook" logCmd "so-playbook-reset" fi