diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 5457a2703..ccf57b2d7 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -19,7 +19,6 @@ { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } }, { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel deleted file mode 100644 index d03cd34e4..000000000 --- a/salt/elasticsearch/files/ingest/import.wel +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description" : "import.wel", - "processors" : [ - { "set": { "field": "event.ingested", "value": "{{ @timestamp }}" } }, - { "set" : { "field" : "@timestamp", "value" : "{{ event.created }}" } }, - { "remove": { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } }, - { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, - { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { "pipeline": { "name": "common" } } - ] -} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth new file mode 100644 index 000000000..3381947eb --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth @@ -0,0 +1,113 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +INFLUX_URL="https://localhost:8086/api/v2" + +. /usr/sbin/so-common + +request() { + curl -skK /opt/so/conf/influxdb/curl.config "$INFLUX_URL/$@" +} + +lookup_org_id() { + response=$(request orgs?org=Security+Onion) + echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id" +} + +ORG_ID=$(lookup_org_id) + +run_flux_query() { + local query=$1 + request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null +} + +read_csv_result() { + local result="$1" + echo "$result" | grep '^,_result,' | head -1 | awk -F',' '{print $NF}' | tr -d '\r\n\t ' +} + +bytes_to_gb() { + local bytes="${1:-0}" + if [[ "$bytes" =~ ^-?[0-9]+$ ]]; then + echo "$bytes" | awk '{printf "%.2f", $1 / 1024 / 1024 / 1024}' + else + echo "0.00" + fi +} + +indexes_query='from(bucket: "telegraf/so_long_term") +|> range(start: -7d) +|> filter(fn: (r) => r._measurement == "elasticsearch_index_size") +|> distinct(column: "_field") +|> keep(columns: ["_field"])' + +indexes_result=$(run_flux_query "$indexes_query") +indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) + +printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" +printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + +for index in $indexes; do + [[ -z "$index" ]] && continue + current_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -4h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + current_result=$(run_flux_query "$current_query") + current_size=$(read_csv_result "$current_result") + current_size=${current_size:-0} + + size_24h_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -25h, stop: -23h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_24h_result=$(run_flux_query "$size_24h_query") + size_24h_ago=$(read_csv_result "$size_24h_result") + size_24h_ago=${size_24h_ago:-$current_size} + + size_7d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -7d8h, stop: -7d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_7d_result=$(run_flux_query "$size_7d_query") + size_7d_ago=$(read_csv_result "$size_7d_result") + size_7d_ago=${size_7d_ago:-$current_size} + + size_30d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -30d8h, stop: -30d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_30d_result=$(run_flux_query "$size_30d_query") + size_30d_ago=$(read_csv_result "$size_30d_result") + size_30d_ago=${size_30d_ago:-$current_size} + + # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. + growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) + + growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) + + growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) + + growth_24h_gb=$(bytes_to_gb "$growth_24h") + growth_7d_gb=$(bytes_to_gb "$growth_7d") + growth_30d_gb=$(bytes_to_gb "$growth_30d") + + # Only results for indices with atleast 1 metric above 0.00 + if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" \ + "$growth_24h" \ + "$index" \ + "$growth_24h_gb" \ + "$growth_7d_gb" \ + "$growth_30d_gb" + fi +done | sort -t'|' -k1,1nr | cut -d'|' -f2- + diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index bff46dd5b..88eea6dc3 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -14,7 +14,7 @@ include: # Install the registry container so-dockerregistry: docker_container.running: - - image: ghcr.io/security-onion-solutions/registry:2.8.3 + - image: ghcr.io/security-onion-solutions/registry:3.0.0 - hostname: so-registry - networks: - sobridge: @@ -25,7 +25,7 @@ so-dockerregistry: - {{ BINDING }} {% endfor %} - binds: - - /opt/so/conf/docker-registry/etc/config.yml:/etc/docker/registry/config.yml:ro + - /opt/so/conf/docker-registry/etc/config.yml:/etc/distribution/config.yml:ro - /opt/so/conf/docker-registry:/var/lib/registry:rw - /nsm/docker-registry/docker:/var/lib/registry/docker:rw - /etc/pki/registry.crt:/etc/pki/registry.crt:ro diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index f1ec6e452..24b9aaba2 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1464,8 +1464,8 @@ soc: autoUpdateEnabled: true playbookImportFrequencySeconds: 86400 playbookImportErrorSeconds: 600 - playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources - playbookRepoBranch: playbooks-stable + playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + playbookRepoBranch: main playbookRepoPath: /opt/sensoroni/playbooks/ playbookPathInRepo: securityonion-normalized salt: