diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 05c47a6c1..902aabaa3 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.10.4" +ELASTIC_AGENT_TARBALL_VERSION="8.14.3" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 27a8a0fd6..a3dd189ad 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,11 @@ esingestconf: - user: 930 - group: 939 +# Remove .fleet_final_pipeline-1 because we are using global@custom now +so-fleet-final-pipeline-remove: + file.absent: + - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1 + # Auto-generate Elasticsearch ingest node pipelines from pillar {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} es_ingest_conf_{{pipeline}}: diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 deleted file mode 100644 index 233cd647b..000000000 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ /dev/null @@ -1,107 +0,0 @@ -{ - "version": 3, - "_meta": { - "managed_by": "fleet", - "managed": true - }, - "description": "Final pipeline for processing all incoming Fleet Agent documents. \n", - "processors": [ - { - "date": { - "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)", - "tag": "truncate-subseconds-event-ingested", - "field": "_ingest.timestamp", - "target_field": "event.ingested", - "formats": [ - "ISO8601" - ], - "output_format": "date_time_no_millis", - "ignore_failure": true - } - }, - { - "remove": { - "description": "Remove any pre-existing untrusted values.", - "field": [ - "event.agent_id_status", - "_security" - ], - "ignore_missing": true - } - }, - { - "set_security_user": { - "field": "_security", - "properties": [ - "authentication_type", - "username", - "realm", - "api_key" - ] - } - }, - { - "script": { - "description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n", - "tag": "agent-id-status", - "source": "boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != 'API_KEY'\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);", - "params": { - "trusted_users": [ - { - "username": "elastic/fleet-server", - "realm": "_service_account" - }, - { - "username": "cloud-internal-agent-server", - "realm": "found" - }, - { - "username": "elastic", - "realm": "reserved" - } - ] - } - } - }, - { - "remove": { - "field": "_security", - "ignore_missing": true - } - }, - { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, - { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, - { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, - { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, - { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, - { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, - { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, - { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, - { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, - { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, - { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } - ], - "on_failure": [ - { - "remove": { - "field": "_security", - "ignore_missing": true, - "ignore_failure": true - } - }, - { - "append": { - "field": "error.message", - "value": [ - "failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}" - ] - } - } - ] -} diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom new file mode 100644 index 000000000..dbf215fb1 --- /dev/null +++ b/salt/elasticsearch/files/ingest/global@custom @@ -0,0 +1,27 @@ +{ + "version": 3, + "_meta": { + "managed_by": "securityonion", + "managed": true + }, + "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n", + "processors": [ + { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, + { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, + { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, + { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, + { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, + { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, + { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, + { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, + { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, + { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, + { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, + { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, + { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, + { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } + ] +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json new file mode 100644 index 000000000..8fc83f9cb --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json @@ -0,0 +1,201 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "default_pipeline": "metrics-fleet_server.agent_status-1.5.0", + "mapping": { + "total_fields": { + "limit": "1000" + } + } + } + }, + "mappings": { + "dynamic": false, + "_source": { + "mode": "synthetic" + }, + "properties": { + "cluster": { + "properties": { + "id": { + "time_series_dimension": true, + "type": "keyword" + } + } + }, + "fleet": { + "properties": { + "agents": { + "properties": { + "offline": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "total": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "updating": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "inactive": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "healthy": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unhealthy": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unenrolled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "enrolled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unhealthy_reason": { + "properties": { + "output": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "input": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "other": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + } + } + }, + "upgrading_step": { + "properties": { + "rollback": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "requested": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "restarting": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "downloading": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "scheduled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "extracting": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "replacing": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "failed": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "watching": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + } + } + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "kibana": { + "properties": { + "uuid": { + "path": "agent.id", + "type": "alias" + }, + "version": { + "path": "agent.version", + "type": "alias" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "fleet_server" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json new file mode 100644 index 000000000..af3323ee9 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json @@ -0,0 +1,102 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "default_pipeline": "metrics-fleet_server.agent_versions-1.5.0", + "mapping": { + "total_fields": { + "limit": "1000" + } + } + } + }, + "mappings": { + "dynamic": false, + "_source": { + "mode": "synthetic" + }, + "properties": { + "cluster": { + "properties": { + "id": { + "time_series_dimension": true, + "type": "keyword" + } + } + }, + "fleet": { + "properties": { + "agent": { + "properties": { + "count": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "version": { + "time_series_dimension": true, + "type": "keyword" + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "kibana": { + "properties": { + "uuid": { + "path": "agent.id", + "type": "alias" + }, + "version": { + "path": "agent.version", + "type": "alias" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "fleet_server" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index 71c40c1ca..b76a0e0f0 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -20,7 +20,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." - for i in .[a-z]* *; + for i in *; do echo $i; retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i" diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index bc503debb..c52b5cb44 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1,2 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.3","id": "8.14.3","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} + diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index 5b4b52560..8177adb5c 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index bc1306673..221eb3799 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -401,6 +401,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 + [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100 true } @@ -420,6 +421,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 [[ "$POSTVERSION" == 2.4.70 ]] && post_to_2.4.80 [[ "$POSTVERSION" == 2.4.80 ]] && post_to_2.4.90 + [[ "$POSTVERSION" == 2.4.90 ]] && post_to_2.4.100 true } @@ -504,6 +506,11 @@ post_to_2.4.90() { POSTVERSION=2.4.90 } +post_to_2.4.100() { + echo "Nothing to apply" + POSTVERSION=2.4.100 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -685,6 +692,11 @@ up_to_2.4.90() { INSTALLEDVERSION=2.4.90 } +up_to_2.4.100() { + # Elastic Update for this release, so download Elastic Agent files + determine_elastic_agent_upgrade + INSTALLEDVERSION=2.4.100 +} add_detection_test_pillars() { if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then