From fcdacc3b0dd838585d96e02a7f193a0f47a4f588 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 29 May 2025 12:21:28 -0500
Subject: [PATCH 1/9] fix system integration time overwrite and delete unused
 ingest pipeline

---
 salt/elasticsearch/files/ingest/global@custom |  1 -
 salt/elasticsearch/files/ingest/import.wel    | 11 -----------
 2 files changed, 12 deletions(-)
 delete mode 100644 salt/elasticsearch/files/ingest/import.wel

diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom
index 5457a2703..ccf57b2d7 100644
--- a/salt/elasticsearch/files/ingest/global@custom
+++ b/salt/elasticsearch/files/ingest/global@custom
@@ -19,7 +19,6 @@
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
     { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
     { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
     { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel
deleted file mode 100644
index d03cd34e4..000000000
--- a/salt/elasticsearch/files/ingest/import.wel
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-  "description" : "import.wel",
-  "processors" : [
-    { "set":           { "field": "event.ingested", "value": "{{ @timestamp }}"  } },
-    { "set" :          { "field" : "@timestamp", "value" : "{{ event.created }}" } },
-    { "remove":        { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } },
-    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon" } },
-    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" } },
-    { "pipeline":      { "name": "common" } }
-  ]
-}

From 1b89cc681883fab7d3b2e70154a43dae59babfd0 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:41:03 -0500
Subject: [PATCH 2/9] so-elasticsearch-index-growth script

---
 .../tools/sbin/so-elasticsearch-index-growth  | 118 ++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth

diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
new file mode 100644
index 000000000..bcaa00fa2
--- /dev/null
+++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
@@ -0,0 +1,118 @@
+#! /bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+INFLUX_URL="https://localhost:8086/api/v2"
+
+. /usr/sbin/so-common
+
+request() {
+    curl -skK /opt/so/conf/influxdb/curl.config "$INFLUX_URL/$@"
+}
+
+lookup_org_id() {
+  response=$(request orgs?org=Security+Onion)
+  echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
+}
+
+ORG_ID=$(lookup_org_id)
+
+run_flux_query() {
+    local query=$1
+    request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
+}
+
+read_csv_result() {
+    local result="$1"
+    echo "$result" | grep '^,_result,' | head -1 | awk -F',' '{print $NF}' | tr -d '\r\n\t '
+}
+
+bytes_to_gb() {
+    local bytes="${1:-0}"
+    if [[ "$bytes" =~ ^-?[0-9]+$ ]]; then
+        echo "$bytes" | awk '{printf "%.2f", $1 / 1024 / 1024 / 1024}'
+    else
+        echo "0.00"
+    fi
+}
+
+indexes_query='from(bucket: "telegraf/so_long_term")
+|> range(start: -7d)
+|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
+|> distinct(column: "_field")
+|> keep(columns: ["_field"])'
+
+indexes_result=$(run_flux_query "$indexes_query")
+indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
+
+printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
+printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
+
+temp_results=$(mktemp)
+
+for index in $indexes; do
+  [[ -z "$index" ]] && continue
+  current_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -4h)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  current_result=$(run_flux_query "$current_query")
+  current_size=$(read_csv_result "$current_result")
+  current_size=${current_size:-0}
+
+  size_24h_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -25h, stop: -23h)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_24h_result=$(run_flux_query "$size_24h_query")
+  size_24h_ago=$(read_csv_result "$size_24h_result")
+  size_24h_ago=${size_24h_ago:-$current_size}
+
+  size_7d_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -8d, stop: -6d)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_7d_result=$(run_flux_query "$size_7d_query")
+  size_7d_ago=$(read_csv_result "$size_7d_result")
+  size_7d_ago=${size_7d_ago:-$current_size}
+
+  size_30d_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -31d, stop: -29d)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_30d_result=$(run_flux_query "$size_30d_query")
+  size_30d_ago=$(read_csv_result "$size_30d_result")
+  size_30d_ago=${size_30d_ago:-$current_size}
+
+  # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
+  growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
+
+  growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
+
+  growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
+
+  growth_24h_gb=$(bytes_to_gb "$growth_24h")
+  growth_7d_gb=$(bytes_to_gb "$growth_7d")
+  growth_30d_gb=$(bytes_to_gb "$growth_30d")
+
+  # Only results for indices with atleast 1 metric above 0.00
+  if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
+    printf "%020.2f|%-50s %15s %15s %15s\n" \
+      "$growth_24h" \
+      "$index" \
+      "$growth_24h_gb" \
+      "$growth_7d_gb" \
+      "$growth_30d_gb" >> "$temp_results"
+  fi
+done
+
+sort -t'|' -k1,1nr "$temp_results" | cut -d'|' -f2-
+rm -f "$temp_results"
+

From 061600fa7ac3d9978945a9aa655d48c1eae84dee Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:55:46 -0500
Subject: [PATCH 3/9] shebang line

---
 salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
index bcaa00fa2..801f64342 100644
--- a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
+++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
@@ -1,4 +1,4 @@
-#! /bin/bash
+#!/bin/bash
 
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at

From 6238a5b3edf048dfa5ba70e0948bb69262ffb4ff Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:31:26 -0500
Subject: [PATCH 4/9] tighten up search timeframe

---
 salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
index 801f64342..7e31a5eed 100644
--- a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
+++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
@@ -74,7 +74,7 @@ for index in $indexes; do
   size_24h_ago=${size_24h_ago:-$current_size}
 
   size_7d_query="from(bucket: \"telegraf/so_long_term\")
-    |> range(start: -8d, stop: -6d)
+    |> range(start: -7d8h, stop: -7d)
     |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
     |> last()
     |> keep(columns: [\"_value\"])"
@@ -83,7 +83,7 @@ for index in $indexes; do
   size_7d_ago=${size_7d_ago:-$current_size}
 
   size_30d_query="from(bucket: \"telegraf/so_long_term\")
-    |> range(start: -31d, stop: -29d)
+    |> range(start: -30d8h, stop: -30d)
     |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
     |> last()
     |> keep(columns: [\"_value\"])"

From 4d6171bde6aeac8bdf1d7e78d2d3580e030fcca7 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 3 Jun 2025 07:32:12 -0500
Subject: [PATCH 5/9] rename script

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 ...elasticsearch-index-growth => so-elasticsearch-indices-growth} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename salt/elasticsearch/tools/sbin/{so-elasticsearch-index-growth => so-elasticsearch-indices-growth} (100%)

diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth
similarity index 100%
rename from salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth
rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth

From d240fca72108b4e8af2d11454cf83567665a6326 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 3 Jun 2025 08:45:04 -0500
Subject: [PATCH 6/9] remove usage of temp file

---
 .../tools/sbin/so-elasticsearch-indices-growth           | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth
index 7e31a5eed..3381947eb 100644
--- a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth
+++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth
@@ -51,8 +51,6 @@ indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | g
 printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
 printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
 
-temp_results=$(mktemp)
-
 for index in $indexes; do
   [[ -z "$index" ]] && continue
   current_query="from(bucket: \"telegraf/so_long_term\")
@@ -109,10 +107,7 @@ for index in $indexes; do
       "$index" \
       "$growth_24h_gb" \
       "$growth_7d_gb" \
-      "$growth_30d_gb" >> "$temp_results"
+      "$growth_30d_gb"
   fi
-done
-
-sort -t'|' -k1,1nr "$temp_results" | cut -d'|' -f2-
-rm -f "$temp_results"
+done | sort -t'|' -k1,1nr | cut -d'|' -f2-
 

From fc9107f1290334fd12fbda3b637cd695c8c22c3e Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Tue, 3 Jun 2025 13:33:30 -0600
Subject: [PATCH 7/9] Updated Playbook Repo Config

The repo and folder have changed. We're splitting out playbooks into their own repo: github.com/security-onion-solutions/securityonion-resources-playbooks.
---
 salt/soc/defaults.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index f1ec6e452..24b9aaba2 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1464,8 +1464,8 @@ soc:
           autoUpdateEnabled: true
           playbookImportFrequencySeconds: 86400
           playbookImportErrorSeconds: 600
-          playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources
-          playbookRepoBranch: playbooks-stable
+          playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
+          playbookRepoBranch: main
           playbookRepoPath: /opt/sensoroni/playbooks/
           playbookPathInRepo: securityonion-normalized
         salt:

From 832d66052e92d9de36a3494bc486dc8e7bdc5c99 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 4 Jun 2025 09:13:54 -0400
Subject: [PATCH 8/9] upgrade registry to 3.0.0

---
 salt/registry/enabled.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls
index bff46dd5b..592afd976 100644
--- a/salt/registry/enabled.sls
+++ b/salt/registry/enabled.sls
@@ -14,7 +14,7 @@ include:
 # Install the registry container
 so-dockerregistry:
   docker_container.running:
-    - image: ghcr.io/security-onion-solutions/registry:2.8.3
+    - image: ghcr.io/security-onion-solutions/registry:3.0.0
     - hostname: so-registry
     - networks:
       - sobridge:

From ab89858d04f61809f8198774c52dca6787c3d5e7 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 4 Jun 2025 17:19:53 -0400
Subject: [PATCH 9/9] update to new config location

---
 salt/registry/enabled.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls
index 592afd976..88eea6dc3 100644
--- a/salt/registry/enabled.sls
+++ b/salt/registry/enabled.sls
@@ -25,7 +25,7 @@ so-dockerregistry:
       - {{ BINDING }}
       {% endfor %}
     - binds:
-      - /opt/so/conf/docker-registry/etc/config.yml:/etc/docker/registry/config.yml:ro
+      - /opt/so/conf/docker-registry/etc/config.yml:/etc/distribution/config.yml:ro
       - /opt/so/conf/docker-registry:/var/lib/registry:rw
       - /nsm/docker-registry/docker:/var/lib/registry/docker:rw
       - /etc/pki/registry.crt:/etc/pki/registry.crt:ro