diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 46a0b8e9d..3b41dd960 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -8,32 +8,58 @@ . /usr/sbin/so-common +elastic_fleet_policy_create() { -# Create ES Token -ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) -printf "ESTOKEN = $ESTOKEN \n" + NAME=$1 + DESC=$2 + FLEETSERVER=$3 -# Add SO-Manager Fleet URL -## This array replaces whatever URLs are currently configured -printf "\n" -curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' -printf "\n\n" + JSON_STRING=$( jq -n \ + --arg NAME "$NAME" \ + --arg DESC "$DESC" \ + --arg FLEETSERVER "$FLEETSERVER" \ + '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":$FLEETSERVER}' + ) + # Create Fleet Policy + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + +} + +elastic_fleet_policy_update() { + + POLICYID=$1 + JSONBLOB=$2 + + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE" +} + +elastic_fleet_integration_create() { + + JSONBLOB=$1 + + #curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSONBLOB" + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSONBLOB" + +} -# Configure certificates mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs -# Add SO-Manager Elasticsearch Ouput +printf "\n### Create ES Token ###\n" +ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) + +### Create Outputs & Fleet URLs ### +printf "\nAdd Manager Elasticsearch Ouput...\n" ESCACRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) JSON_STRING=$( jq -n \ --arg ESCACRT "$ESCACRT" \ - '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) + '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" +printf "\nCreate Logstash Output if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} -# Create Logstash Output payload LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) @@ -41,34 +67,42 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}' + '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055", "{{ GLOBALS.manager }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}' ) - -# Add Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" {%- endif %} -# Add Elastic Fleet Integrations +printf "\nAdd SO-Manager Fleet URL\n" +## This array replaces whatever URLs are currently configured +curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220", "https://{{ GLOBALS.manager }}:8220"]}' +printf "\n\n" -# Add Elastic Fleet Server Agent Policy -#curl -vv -K /opt/so/conf/elasticsearch/curl.config -L \ -#-X POST "localhost:5601/api/fleet/agent_policies" \ -#-H 'kbn-xsrf: true' -H 'Content-Type: application/json' \ -#-d '{"name":"SO-Manager","id":"so-manager","description":"SO Manager Fleet Server Policy","namespace":"default","monitoring_enabled":["logs"],"has_fleet_server":true}' -# Add Agent Policy - SOS Grid Nodes -#curl -vv -K /opt/so/conf/elasticsearch/curl.config -L \ -#-X POST "localhost:5601/api/fleet/agent_policies" \ -#-H 'kbn-xsrf: true' -H 'Content-Type: application/json' \ -#-d '{"name":"SO-Grid","id":"so-grid","description":"SO Grid Endpoint Policy","namespace":"default","monitoring_enabled":["logs"]}' +### Create Policies & Associated Integration Configuration ### -# Add Agent Policy - Default endpoints -#curl -vv -K /opt/so/conf/elasticsearch/curl.config -L \ -#-X POST "localhost:5601/api/fleet/agent_policies" \ -#-H 'kbn-xsrf: true' -H 'Content-Type: application/json' \ -#-d '{"name":"Endpoints-Initalization","id":"endpoints","description":"Initial Endpoint Policy","namespace":"default","monitoring_enabled":["logs"]}' +# Manager Fleet Server Host +elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" | jq +#elastic_fleet_policy_update "FleetServer_{{ GLOBALS.hostname }}" "@FleeServerHost_Fixup" +# Initial Endpoints +elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" | jq +for INTEGRATION in /opt/so/saltstack/opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json +do + elastic_fleet_integration_create "@$INTEGRATION" | jq +done + +# Grid Nodes +elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false" +for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json +do + elastic_fleet_integration_create "@$INTEGRATION" | jq +done + + +### Finalization ### + +# Query for Enrollment Tokens for default policies ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-default")) | .api_key') GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key') @@ -80,7 +114,6 @@ printf '%s\n'\ " es_token: '$ESTOKEN'"\ " endpoints_enrollment: '$ENDPOINTSENROLLMENTOKEN'"\ " grid_enrollment: '$GRIDNODESENROLLMENTOKEN'"\ - " url: '{{ GLOBALS.manager_ip }}'"\ "" >> "$pillar_file" #Store Grid Nodes Enrollment token in Global pillar @@ -92,17 +125,32 @@ printf '%s\n'\ # Call Elastic-Fleet Salt State salt-call state.apply elasticfleet queue=True -# Load Elastic Fleet integrations -/usr/sbin/so-elastic-fleet-integration-policy-load + + + + + +### DEPRECATED # Temp +# Configure certificates +mkdir -p /opt/so/conf/elastic-fleet/certs +cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs +cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs + wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.7.0/so-elastic-agent-8.7.0-darwin-x86_64.tar.gz wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.7.0/so-elastic-agent-8.7.0-linux-x86_64.tar.gz wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.7.0/so-elastic-agent-8.7.0-windows-x86_64.tar.gz -#git clone -b 2.4-so-elastic-agent https://github.com/Security-Onion-Solutions/securityonion-image.git -#cd securityonion-image/so-elastic-agent-builder -#docker build -t so-elastic-agent-builder . - so-elastic-agent-gen-installers salt-call state.apply elasticfleet.install_agent_grid queue=True + + +#Temp Fixup for Fleet Server Host Output +JSON_STRING_UPDATE=$( jq -n \ + --arg NAME "FleetServer_$MINIONID" \ + --arg DESC "Fleet Server - $MINIONID" \ + '{"name":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"data_output_id":"so-manager_elasticsearch"}' + ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_$MINIONID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE" + diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index 7b698a119..543315844 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -220,7 +220,6 @@ function add_sensor_to_minion() { function create_fleet_policy() { - MINIONID="sa-29-261-jb_standalone" JSON_STRING=$( jq -n \ --arg NAME "FleetServer_$MINIONID" \ --arg DESC "Fleet Server - $MINIONID" \ diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json new file mode 100644 index 000000000..71d2345a5 --- /dev/null +++ b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json @@ -0,0 +1,20 @@ +{ + "package": { + "name": "osquery_manager", + "version": "1.6.0" + }, + "name": "osquery-endpoints", + "namespace": "default", + "policy_id": "endpoints-initial", + "inputs": { + "osquery_manager-osquery": { + "enabled": true, + "streams": { + "osquery_manager.result": { + "enabled": true, + "vars": {} + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json new file mode 100644 index 000000000..1b918be1a --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.1" + }, + "name": "idh-logs", + "namespace": "so", + "description": "IDH integration", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/idh/opencanary.log" + ], + "data_stream.dataset": "idh", + "tags": [], + "processors": "\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- drop_fields:\n when:\n equals:\n logtype: \"1001\"\n fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n ignore_missing: true\n- rename:\n fields:\n - from: \"src_host\"\n to: \"source.ip\"\n - from: \"src_port\"\n to: \"source.port\"\n - from: \"dst_host\"\n to: \"destination.host\"\n - from: \"dst_port\"\n to: \"destination.port\"\n ignore_missing: true\n- convert:\n fields:\n - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n ignore_missing: true\n- drop_fields:\n fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n target: event\n fields:\n category: host\n module: opencanary", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json new file mode 100644 index 000000000..87c4fc82c --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.0" + }, + "name": "kratos-logs", + "namespace": "so", + "description": "Kratos logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/kratos/kratos.log" + ], + "data_stream.dataset": "kratos", + "tags": [], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", + "custom": "pipeline: kratos" + } + } + } + } + } +} diff --git a/salt/elasticfleet/init.sls b/salt/elasticfleet/init.sls index b457ccf5b..2c3b5cfce 100644 --- a/salt/elasticfleet/init.sls +++ b/salt/elasticfleet/init.sls @@ -8,7 +8,7 @@ # These values are generated during node install and stored in minion pillar {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:server:es_token','') %} -{% set FLEETSERVERPOLICY = salt['pillar.get']('elasticfleet:server:server_policy','so-manager') %} +#{% set FLEETSERVERPOLICY = salt['pillar.get']('elasticfleet:server:server_policy','so-manager') %} #{% set FLEETURL = salt['pillar.get']('elasticfleet:server:url') %} # Add EA Group