CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13402 from Security-Onion-Solutions/foxtrot

Browse Source 
Elastic 8.14.3
This commit is contained in:
weslambert
2024-07-30 09:28:19 -04:00
committed by GitHub
parent 685df9e5ea 6d008546f1
commit b466d83625
10 changed files with 352 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
salt/common/tools/sbin/so-common
View File

@@ -8,7 +8,7 @@
# Elastic agent is not managed by salt. Because of this we must store this base information in a # Elastic agent is not managed by salt. Because of this we must store this base information in a
# script that accompanies the soup system. Since so-common is one of those special soup files, # script that accompanies the soup system. Since so-common is one of those special soup files,
# and since this same logic is required during installation, it's included in this file. # and since this same logic is required during installation, it's included in this file.
ELASTIC_AGENT_TARBALL_VERSION="8.10.4" ELASTIC_AGENT_TARBALL_VERSION="8.14.3"
ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"

5
salt/elasticsearch/config.sls
View File

@@ -118,6 +118,11 @@ esingestconf:
- user: 930 - user: 930
- group: 939 - group: 939
# Remove .fleet_final_pipeline-1 because we are using global@custom now
so-fleet-final-pipeline-remove:
file.absent:
- name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1
# Auto-generate Elasticsearch ingest node pipelines from pillar # Auto-generate Elasticsearch ingest node pipelines from pillar
{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
es_ingest_conf_{{pipeline}}: es_ingest_conf_{{pipeline}}:

107
salt/elasticsearch/files/ingest/.fleet_final_pipeline-1
View File

@@ -1,107 +0,0 @@
{
"version": 3,
"_meta": {
"managed_by": "fleet",
"managed": true
},
"description": "Final pipeline for processing all incoming Fleet Agent documents. \n",
"processors": [
{
"date": {
"description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)",
"tag": "truncate-subseconds-event-ingested",
"field": "_ingest.timestamp",
"target_field": "event.ingested",
"formats": [
"ISO8601"
],
"output_format": "date_time_no_millis",
"ignore_failure": true
}
},
{
"remove": {
"description": "Remove any pre-existing untrusted values.",
"field": [
"event.agent_id_status",
"_security"
],
"ignore_missing": true
}
},
{
"set_security_user": {
"field": "_security",
"properties": [
"authentication_type",
"username",
"realm",
"api_key"
]
}
},
{
"script": {
"description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n",
"tag": "agent-id-status",
"source": "boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != 'API_KEY'\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);",
"params": {
"trusted_users": [
{
"username": "elastic/fleet-server",
"realm": "_service_account"
},
{
"username": "cloud-internal-agent-server",
"realm": "found"
},
{
"username": "elastic",
"realm": "reserved"
}
]
}
}
},
{
"remove": {
"field": "_security",
"ignore_missing": true
}
},
{ "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } },
{ "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } },
{ "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
{ "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
{ "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } },
{ "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
{ "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
{ "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
{ "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
{ "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
{ "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
{ "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
{ "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } },
{ "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
{ "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
{ "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
{ "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } }
],
"on_failure": [
{
"remove": {
"field": "_security",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"append": {
"field": "error.message",
"value": [
"failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"
]
}
}
]
}

27
salt/elasticsearch/files/ingest/global@custom Normal file
View File

@@ -0,0 +1,27 @@
{
"version": 3,
"_meta": {
"managed_by": "securityonion",
"managed": true
},
"description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
"processors": [
{ "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } },
{ "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } },
{ "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
{ "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
{ "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } },
{ "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
{ "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
{ "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
{ "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
{ "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
{ "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
{ "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
{ "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } },
{ "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
{ "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
{ "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
{ "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } }
]
}

201
salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json Normal file
View File

@@ -0,0 +1,201 @@
{
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "metrics"
},
"default_pipeline": "metrics-fleet_server.agent_status-1.5.0",
"mapping": {
"total_fields": {
"limit": "1000"
}
}
}
},
"mappings": {
"dynamic": false,
"_source": {
"mode": "synthetic"
},
"properties": {
"cluster": {
"properties": {
"id": {
"time_series_dimension": true,
"type": "keyword"
}
}
},
"fleet": {
"properties": {
"agents": {
"properties": {
"offline": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"total": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"updating": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"inactive": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"healthy": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"unhealthy": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"unenrolled": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"enrolled": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"unhealthy_reason": {
"properties": {
"output": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"input": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"other": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
}
}
},
"upgrading_step": {
"properties": {
"rollback": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"requested": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"restarting": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"downloading": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"scheduled": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"extracting": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"replacing": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"failed": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"watching": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
}
}
}
}
}
}
},
"agent": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"@timestamp": {
"ignore_malformed": false,
"type": "date"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
},
"type": {
"type": "constant_keyword"
},
"dataset": {
"type": "constant_keyword"
}
}
},
"kibana": {
"properties": {
"uuid": {
"path": "agent.id",
"type": "alias"
},
"version": {
"path": "agent.version",
"type": "alias"
}
}
}
}
}
},
"_meta": {
"package": {
"name": "fleet_server"
},
"managed_by": "fleet",
"managed": true
}
}

102
salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json Normal file
View File

@@ -0,0 +1,102 @@
{
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "metrics"
},
"default_pipeline": "metrics-fleet_server.agent_versions-1.5.0",
"mapping": {
"total_fields": {
"limit": "1000"
}
}
}
},
"mappings": {
"dynamic": false,
"_source": {
"mode": "synthetic"
},
"properties": {
"cluster": {
"properties": {
"id": {
"time_series_dimension": true,
"type": "keyword"
}
}
},
"fleet": {
"properties": {
"agent": {
"properties": {
"count": {
"time_series_metric": "gauge",
"meta": {},
"type": "long"
},
"version": {
"time_series_dimension": true,
"type": "keyword"
}
}
}
}
},
"agent": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"@timestamp": {
"ignore_malformed": false,
"type": "date"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
},
"type": {
"type": "constant_keyword"
},
"dataset": {
"type": "constant_keyword"
}
}
},
"kibana": {
"properties": {
"uuid": {
"path": "agent.id",
"type": "alias"
},
"version": {
"path": "agent.version",
"type": "alias"
}
}
}
}
}
},
"_meta": {
"package": {
"name": "fleet_server"
},
"managed_by": "fleet",
"managed": true
}
}

2
salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines
View File

@@ -20,7 +20,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then
cd ${ELASTICSEARCH_INGEST_PIPELINES} cd ${ELASTICSEARCH_INGEST_PIPELINES}
echo "Loading pipelines..." echo "Loading pipelines..."
for i in .[a-z]* *; for i in *;
do do
echo $i; echo $i;
retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i" retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i"

3
salt/kibana/files/config_saved_objects.ndjson
View File

@@ -1 +1,2 @@
{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} {"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.3","id": "8.14.3","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

2
salt/kibana/tools/sbin_jinja/so-kibana-config-load
View File

@@ -63,7 +63,7 @@ update() {
IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))'
for i in "${LINES[@]}"; do for i in "${LINES[@]}"; do
RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
done done

12
salt/manager/tools/sbin/soup
View File

@@ -401,6 +401,7 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70
[[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80
[[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90
[[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100
true true
} }
@@ -420,6 +421,7 @@ postupgrade_changes() {
[[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70
[[ "$POSTVERSION" == 2.4.70 ]] && post_to_2.4.80 [[ "$POSTVERSION" == 2.4.70 ]] && post_to_2.4.80
[[ "$POSTVERSION" == 2.4.80 ]] && post_to_2.4.90 [[ "$POSTVERSION" == 2.4.80 ]] && post_to_2.4.90
[[ "$POSTVERSION" == 2.4.90 ]] && post_to_2.4.100
true true
} }
@@ -504,6 +506,11 @@ post_to_2.4.90() {
POSTVERSION=2.4.90 POSTVERSION=2.4.90
} }
post_to_2.4.100() {
echo "Nothing to apply"
POSTVERSION=2.4.100
}
repo_sync() { repo_sync() {
echo "Sync the local repo." echo "Sync the local repo."
su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -685,6 +692,11 @@ up_to_2.4.90() {
INSTALLEDVERSION=2.4.90 INSTALLEDVERSION=2.4.90
} }
up_to_2.4.100() {
# Elastic Update for this release, so download Elastic Agent files
determine_elastic_agent_upgrade
INSTALLEDVERSION=2.4.100
}
add_detection_test_pillars() { add_detection_test_pillars() {
if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then