From 1fb6cf7bfeaa3d6777b1683be3117dc4d959dbd9 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 15:59:49 +0000 Subject: [PATCH 1/8] Add Elastic Fleet to allowed states for Import Mode --- salt/allowed_states.map.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 2f6cc60a0..823b7b647 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -107,7 +107,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'elastic-fleet' ], 'so-manager': [ 'salt.master', From 6a803dfe356cf4524f47dcffe8c5e80bacdfaf20 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:01:03 +0000 Subject: [PATCH 2/8] Add Elastic Fleet to top file configuration for Import Mode --- salt/top.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/top.sls b/salt/top.sls index e29d3b081..4b8531f4d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -344,6 +344,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': - match: compound From a9a119f1ab820a6b97901f32f94fcf355cc9a31c Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:02:27 +0000 Subject: [PATCH 3/8] Add Elasticsearch output to 'so-elastic-fleet-setup' for Import Mode --- salt/common/tools/sbin/so-elastic-fleet-setup | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 4e3d7639c..c945aeec3 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,10 +19,22 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" -# Create Logstash Output payload +# Configure certificates mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs + +{% if grains.role == 'so-import' %} +# Add SO-Manager Elasticsearch Ouput +ESCACRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) +JSON_STRING=$( jq -n \ + --arg ESCACRT "$ESCACRT" \ + '{"name":"so-manager_elasticsearch2","id":"so-manager_elasticsearch2","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +printf "\n\n" + +{% else %} +# Create Logstash Output payload LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) @@ -30,12 +42,13 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' + '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' ) # Add SO-Manager Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" +{%- endif %} # Add Elastic Fleet Integrations From 8051fc70eba77529b9fba1ae60a16dab0fa488e4 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:03:59 +0000 Subject: [PATCH 4/8] Temporarily disable the loading of the RITA package policy --- .../tools/sbin/so-elastic-fleet-integration-policy-load | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index b3148f7aa..f409d9a17 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -66,10 +66,10 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS echo # RITA Logs -echo -echo "Setting up RITA package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' -echo +#echo +#echo "Setting up RITA package policy..." +#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' +#echo # Elasticsearch logs echo From 43ffcb1d6382a4ac2d05b281e57d5a99bad0a8c2 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:05:16 +0000 Subject: [PATCH 5/8] Allow setup to set up Elastic Fleet for Import Mode --- setup/so-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 86e9f23d0..8aa34d04f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -587,9 +587,9 @@ if ! [[ -f $install_opt_file ]]; then add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" + title "Setting up Elastic Fleet" + logCmd "so-elastic-fleet-setup" if [[ ! $is_import ]]; then - title "Setting up Elastic Fleet" - logCmd "so-elastic-fleet-setup" title "Setting up Playbook" logCmd "so-playbook-reset" fi From 7d68ef0e8b676236e9ccb9913ce18480d7c6015b Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:07:31 +0000 Subject: [PATCH 6/8] Add Elastic Agent and Fleet to firewall configuration for Import Mode --- salt/firewall/assigned_hostgroups.map.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 6e249ec93..0f16228b2 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -436,6 +436,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elastic_agent_control }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -453,6 +454,10 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + elastic_agent_endpoint: + portgroups: + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} analyst: portgroups: - {{ portgroups.nginx }} From f1db1bc2733b57e4b84c5abaa3312e037fd85bb6 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:12:06 +0000 Subject: [PATCH 7/8] Ensure Kratos events are sent to a data stream instead of an index --- salt/common/tools/sbin/so-image-common | 4 +++- salt/elasticsearch/defaults.yaml | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 3851d8b4a..aa2292b22 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -36,7 +36,9 @@ container_list() { "so-steno" "so-suricata" "so-telegraf" - "so-zeek" + "so-zeek" + "so-elastic-agent" + "so-elastic-agent-builder" ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 11b445c62..3d1182255 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -2677,6 +2677,9 @@ elasticsearch: delete: 365 index_sorting: False index_template: + data_stream: + hidden: false + allow_custom_routing: false index_patterns: - logs-kratos-so* template: From b381c5424e38439448fb3574e568a3d7eac39adf Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:13:23 +0000 Subject: [PATCH 8/8] Remove extra whitespace after 'so-elastic-agent-builder' line in 'so-image-common' --- salt/common/tools/sbin/so-image-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index aa2292b22..945d3f4ed 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -38,7 +38,7 @@ container_list() { "so-telegraf" "so-zeek" "so-elastic-agent" - "so-elastic-agent-builder" + "so-elastic-agent-builder" ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=(