From 510c7a0c19988cefe43f8b888f34afde0f045042 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 12 May 2025 10:48:12 -0400 Subject: [PATCH 01/45] Update 2-4.yml --- .github/DISCUSSION_TEMPLATE/2-4.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/DISCUSSION_TEMPLATE/2-4.yml b/.github/DISCUSSION_TEMPLATE/2-4.yml index fbd4d2c22..d192a0eec 100644 --- a/.github/DISCUSSION_TEMPLATE/2-4.yml +++ b/.github/DISCUSSION_TEMPLATE/2-4.yml @@ -28,6 +28,7 @@ body: - 2.4.140 - 2.4.141 - 2.4.150 + - 2.4.160 - Other (please provide detail below) validations: required: true From fd00a4db85cb7a30f25c946554966c169694fd20 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 12 May 2025 10:48:52 -0400 Subject: [PATCH 02/45] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 316ab4cee..3fe7dce73 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.150 +2.4.160 From 82562f89f6e80f88cb1f109a60a040f6eccdeea3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 14 May 2025 09:23:35 -0400 Subject: [PATCH 03/45] improve consistency --- salt/common/tools/sbin/so-common | 3 ++- salt/common/tools/sbin/so-common-status-check | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index e46eaac69..3e05614d6 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -299,7 +299,8 @@ fail() { get_agent_count() { if [ -f /opt/so/log/agents/agentstatus.log ]; then - AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}') + AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}' | sed 's/,//') + [[ -z "$AGENTCOUNT" ]] && AGENTCOUNT="0" else AGENTCOUNT=0 fi diff --git a/salt/common/tools/sbin/so-common-status-check b/salt/common/tools/sbin/so-common-status-check index 4aa981123..cbef7309e 100644 --- a/salt/common/tools/sbin/so-common-status-check +++ b/salt/common/tools/sbin/so-common-status-check @@ -45,7 +45,7 @@ def check_for_fps(): result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE) if result.returncode == 0: fps = 1 - except FileNotFoundError: + except: fn = '/proc/sys/crypto/' + feat_full + '_enabled' try: with open(fn, 'r') as f: From 69e90e1e70c39d2efcc1e4850e6ba9e2ca64cd4e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 14 May 2025 13:41:08 -0400 Subject: [PATCH 04/45] Update soup Souper Duper! --- salt/manager/tools/sbin/soup | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 7022bf5c0..8f68445c3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -579,6 +579,11 @@ post_to_2.4.150() { POSTVERSION=2.4.150 } +post_to_2.4.160() { + echo "Nothing to apply" + POSTVERSION=2.4.160 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -816,6 +821,12 @@ up_to_2.4.150() { INSTALLEDVERSION=2.4.150 } +up_to_2.4.160() { + echo "Nothing to do for 2.4.160" + + INSTALLEDVERSION=2.4.160 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls From 5c8460fd2610553cdcf1ac6bebf092a5af0d1d43 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 14 May 2025 13:47:26 -0400 Subject: [PATCH 05/45] Update soup --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8f68445c3..987c599f1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -417,6 +417,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.130 ]] && up_to_2.4.140 [[ "$INSTALLEDVERSION" == 2.4.140 ]] && up_to_2.4.141 [[ "$INSTALLEDVERSION" == 2.4.141 ]] && up_to_2.4.150 + [[ "$INSTALLEDVERSION" == 2.4.150 ]] && up_to_2.4.160 true } @@ -444,6 +445,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.130 ]] && post_to_2.4.140 [[ "$POSTVERSION" == 2.4.140 ]] && post_to_2.4.141 [[ "$POSTVERSION" == 2.4.141 ]] && post_to_2.4.150 + [[ "$POSTVERSION" == 2.4.150 ]] && post_to_2.4.160 true } From 78b7068638f33b577787a796d1bd521decec466b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 30 Apr 2025 13:50:20 -0700 Subject: [PATCH 06/45] Playbook Settings Map a folder from the manager's soc config folder to soc's sensoroni folder for storing the playbook repo. Added playbook module section with default values. --- salt/soc/defaults.yaml | 8 ++++++++ salt/soc/enabled.sls | 1 + 2 files changed, 9 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 242050f98..fe190ea69 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1456,6 +1456,14 @@ soc: org: Security Onion bucket: telegraf/so_short_term verifyCert: false + playbook: + autoUpdateEnabled: true + playbookImportFrequencySeconds: 86400 + playbookImportErrorSeconds: 600 + playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources + playbookRepoBranch: playbook-stable + playbookRepoPath: /opt/sensoroni/playbooks/ + playbookPathInRepo: playbook/dev salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index d687289b5..4b39d036e 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -35,6 +35,7 @@ so-soc: - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/ai_summary_repos:/opt/sensoroni/ai_summary_repos:rw - /opt/so/conf/navigator/layers/:/opt/sensoroni/navigator/:rw + - /opt/so/conf/soc/playbooks/:/opt/sensoroni/playbooks/:rw {% if SOCMERGED.telemetryEnabled and not GLOBALS.airgap %} - /opt/so/conf/soc/analytics.js:/opt/sensoroni/html/js/analytics.js:ro {% endif %} From 9022dc24fb11bd061fc345d328af04f08768d043 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 May 2025 08:19:08 -0400 Subject: [PATCH 07/45] Add Parsing for Playbooks --- salt/elasticsearch/files/ingest/global@custom | 4 +- .../elasticsearch/files/ingest/suricata.alert | 3 +- .../files/ingest/suricata.common | 7 ++ salt/soc/files/soc/sigma_so_pipeline.yaml | 97 ++++++++++++++++++- 4 files changed, 106 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 993c08373..54b96be18 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -22,8 +22,10 @@ { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, + { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, + {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}}, + {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}}, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index 9dd97e190..afdc890dd 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -9,6 +9,7 @@ { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, + { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "tag": "suricata" } }, { "pipeline": { "name": "common.nids" } } ] -} +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 8143882c7..578fad17e 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -18,6 +18,13 @@ { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, + {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}}, + { + "script": { + "source": "boolean isPrivate(def ip) { if (ip == null) return false; if (ip.startsWith('10.')) return true; if (ip.startsWith('192.168.')) return true; if (ip.startsWith('172.')) { String[] parts = ip.split('\\\\.'); if (parts.length > 1) { int second = Integer.parseInt(parts[1]); if (second >= 16 && second <= 31) return true; } } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx.containsKey(field) && ctx[field].containsKey('ip') ? ctx[field].ip : null; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); String netField = isPrivate(ip) ? \"private\" : \"public\"; if (!ctx.network.containsKey(netField)) { ctx.network[netField] = new HashMap(); } if (!ctx.network[netField].containsKey(\"ip\")) { ctx.network[netField].ip = new ArrayList(); } if (!ctx.network[netField].ip.contains(ip)) { ctx.network[netField].ip.add(ip); } } }", + "ignore_failure": true + } + }, { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } ] } diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 48e9e1215..88abcc200 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -1,6 +1,45 @@ name: Security Onion Baseline Pipeline priority: 90 transformations: +vars: + document_id: + - '{soc_id}' + hostname: + - '{event_data.host.name}' + ProcessGuid: + - '{event_data.process.entity_id}' + User: + - '{user.name}' + private_ip: + - '{network.private.ip}' + public_ip: + - '{network.public.ip}' + related_ip: + - '{event_data.related.ip}' + related.hosts: + - '{event_data.related.hosts' + CurrentDirectory: + - '{event_data.process.working_directory}' + ParentProcessGuid: + - '{ParentProcessGuid}' + Image: + - '{process.executable}' + community_id: + - '{network.community_id}' +transformations: + - type: value_placeholders + include: + - 'community_id' + - 'document_id' + - 'ProcessGuid' + - 'hostname' + - 'User' + - 'CurrentDirectory' + - 'ParentProcessGuid' + - 'Image' + - 'related_ip' + - 'private_ip' + - 'public_ip' - id: baseline_field_name_mapping type: field_name_mapping mapping: @@ -12,12 +51,27 @@ transformations: sid: rule.uuid answer: answers query: dns.query.name - src_ip: source.ip.keyword + src_ip: source.ip src_port: source.port - dst_ip: destination.ip.keyword + dst_ip: destination.ip dst_port: destination.port winlog.event_data.User: user.name logtype: event.code # OpenCanary + ## Start Temp Linux Mappings ## + ProcessGuid: process.entity_id + ProcessId: process.pid + Image: process.executable + CommandLine: process.command_line + CurrentDirectory: process.working_directory + ParentProcessGuid: process.parent.entity_id + ParentProcessId: process.parent.pid" + ParentImage: process.parent.executable + ParentCommandLine: process.parent.command_line + ## End Temp Linux Mappings ## + rule.type: event.module + related_ip: related.ip + community_id: network.community_id + event_dataset: event.dataset # Maps "opencanary" product to SO IDH logs - id: opencanary_idh_add-fields type: add_condition @@ -126,4 +180,41 @@ transformations: event.type: 'creation' rule_conditions: - type: logsource - category: file_event \ No newline at end of file + category: file_event + category: file_event + # Maps network_connection rules to endpoint network creation logs + # This is an OS-agnostic mapping, to account for logs that don't specify source OS + - id: endpoint_network_connection_add-fields + type: add_condition + conditions: + event.category: 'network' + event.type: 'start' + rule_conditions: + - type: logsource + category: network_connection + # Maps "alert" category to SO Alert events + - id: alert_so_add-fields + type: add_condition + conditions: + tags: 'alert' + rule_conditions: + - type: logsource + category: alert + # Maps "network + connection" to SO connection logs + - id: network_connection_so_add-fields + type: add_condition + conditions: + tags: 'conn' + rule_conditions: + - type: logsource + category: network + service: connection + # Maps "network + dns" to SO DNS logs + - id: network_dns_so_add-fields + type: add_condition + conditions: + tags: 'dns' + rule_conditions: + - type: logsource + category: network + service: dns From 0542c7713773a7efe73f5914a33315f05a0cdc8c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 14 May 2025 16:35:09 -0400 Subject: [PATCH 08/45] Remove wip config --- salt/elasticsearch/files/ingest/global@custom | 2 -- salt/elasticsearch/files/ingest/suricata.alert | 1 - salt/elasticsearch/files/ingest/suricata.common | 7 ------- 3 files changed, 10 deletions(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 54b96be18..d7a331bc2 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -24,8 +24,6 @@ { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}}, - {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}}, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index afdc890dd..71e346728 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -9,7 +9,6 @@ { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, - { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "tag": "suricata" } }, { "pipeline": { "name": "common.nids" } } ] } \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 578fad17e..8143882c7 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -18,13 +18,6 @@ { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, - {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}}, - { - "script": { - "source": "boolean isPrivate(def ip) { if (ip == null) return false; if (ip.startsWith('10.')) return true; if (ip.startsWith('192.168.')) return true; if (ip.startsWith('172.')) { String[] parts = ip.split('\\\\.'); if (parts.length > 1) { int second = Integer.parseInt(parts[1]); if (second >= 16 && second <= 31) return true; } } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx.containsKey(field) && ctx[field].containsKey('ip') ? ctx[field].ip : null; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); String netField = isPrivate(ip) ? \"private\" : \"public\"; if (!ctx.network.containsKey(netField)) { ctx.network[netField] = new HashMap(); } if (!ctx.network[netField].containsKey(\"ip\")) { ctx.network[netField].ip = new ArrayList(); } if (!ctx.network[netField].ip.contains(ip)) { ctx.network[netField].ip.add(ip); } } }", - "ignore_failure": true - } - }, { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } ] } From df103b3dca40348fdb78aef4f85cef7ff66f116f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 14 May 2025 16:36:59 -0400 Subject: [PATCH 09/45] Spacing --- salt/elasticsearch/files/ingest/global@custom | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index d7a331bc2..828ee0a03 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -22,7 +22,7 @@ { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, + { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] From 689db57f5fcd644a3091a9289716d6ba2a25ab85 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 16 May 2025 10:05:38 -0500 Subject: [PATCH 10/45] logstash isn't running on receivers or manager when kafka is the global.pipeline --- salt/telegraf/etc/telegraf.conf | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index f5d331209..e3a286fe4 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -241,18 +241,28 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false -{% if grains.role in ['so-searchnode','so-standalone','so-manager', 'so-managersearch', 'so-heavynode', 'so-receiver'] -%} +{%- if GLOBALS.pipeline == "KAFKA" %} +{%- if grains.role in ['so-searchnode','so-standalone', 'so-managersearch', 'so-heavynode'] -%} [[inputs.logstash]] url = "http://localhost:9600" collect = ["pipelines"] username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" +{%- endif %} +{%- else %} +{%- if grains.role in ['so-searchnode','so-standalone','so-manager', 'so-managersearch', 'so-heavynode', 'so-receiver'] -%} +[[inputs.logstash]] + url = "http://localhost:9600" + collect = ["pipelines"] + username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" + password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" +{%- endif %} {%- endif %} {% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} [[inputs.jolokia2_agent]] name_prefix= "kafka_" - urls = ["http://localhost:8778/jolokia"] + urls = ["http://{{ NODEIP }}:8778/jolokia"] [[inputs.jolokia2_agent.metric]] name = "topics" From 870a9ff80cb0618c16105bc5e0de7068cfde96de Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 16 May 2025 10:24:09 -0500 Subject: [PATCH 11/45] dedup --- salt/telegraf/etc/telegraf.conf | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index e3a286fe4..b358c178f 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -241,22 +241,17 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false -{%- if GLOBALS.pipeline == "KAFKA" %} -{%- if grains.role in ['so-searchnode','so-standalone', 'so-managersearch', 'so-heavynode'] -%} +{%- set logstash_metrics_roles = ['so-searchnode','so-standalone','so-managersearch','so-heavynode'] %} +{%- if GLOBALS.pipeline != "KAFKA" %} +{%- set logstash_metrics_roles = logstash_metrics_roles + ['so-manager', 'so-receiver'] %} +{%- endif %} + +{%- if grains.role in logstash_metrics_roles %} [[inputs.logstash]] url = "http://localhost:9600" collect = ["pipelines"] username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" -{%- endif %} -{%- else %} -{%- if grains.role in ['so-searchnode','so-standalone','so-manager', 'so-managersearch', 'so-heavynode', 'so-receiver'] -%} -[[inputs.logstash]] - url = "http://localhost:9600" - collect = ["pipelines"] - username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" - password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" -{%- endif %} {%- endif %} {% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} From b55cb257b67558a57f74c672aa913cf9a5860919 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 May 2025 13:25:27 -0400 Subject: [PATCH 12/45] Add parsing for Playbook --- salt/elasticsearch/files/ingest/global@custom | 2 + .../elasticsearch/files/ingest/suricata.alert | 1 + .../files/ingest/suricata.common | 7 ++ salt/soc/files/soc/sigma_so_pipeline.yaml | 92 ++++++++++--------- salt/zeek/defaults.yaml | 1 + .../securityonion/community-id-extended.zeek | 40 ++++++++ 6 files changed, 102 insertions(+), 41 deletions(-) create mode 100644 salt/zeek/policy/securityonion/community-id-extended.zeek diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 828ee0a03..85154b908 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -24,6 +24,8 @@ { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, + {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}}, + {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}}, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index 71e346728..3d0241e48 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -9,6 +9,7 @@ { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, + { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } }, { "pipeline": { "name": "common.nids" } } ] } \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 8143882c7..102b5dac8 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -18,6 +18,13 @@ { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, + {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}}, + { + "script": { + "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }", + "ignore_failure": false + } + }, { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } ] } diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 88abcc200..5cee0cfd3 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -1,45 +1,6 @@ name: Security Onion Baseline Pipeline priority: 90 transformations: -vars: - document_id: - - '{soc_id}' - hostname: - - '{event_data.host.name}' - ProcessGuid: - - '{event_data.process.entity_id}' - User: - - '{user.name}' - private_ip: - - '{network.private.ip}' - public_ip: - - '{network.public.ip}' - related_ip: - - '{event_data.related.ip}' - related.hosts: - - '{event_data.related.hosts' - CurrentDirectory: - - '{event_data.process.working_directory}' - ParentProcessGuid: - - '{ParentProcessGuid}' - Image: - - '{process.executable}' - community_id: - - '{network.community_id}' -transformations: - - type: value_placeholders - include: - - 'community_id' - - 'document_id' - - 'ProcessGuid' - - 'hostname' - - 'User' - - 'CurrentDirectory' - - 'ParentProcessGuid' - - 'Image' - - 'related_ip' - - 'private_ip' - - 'public_ip' - id: baseline_field_name_mapping type: field_name_mapping mapping: @@ -64,14 +25,17 @@ transformations: CommandLine: process.command_line CurrentDirectory: process.working_directory ParentProcessGuid: process.parent.entity_id - ParentProcessId: process.parent.pid" + ParentProcessId: process.parent.pid ParentImage: process.parent.executable ParentCommandLine: process.parent.command_line + User: user.name ## End Temp Linux Mappings ## + document_id: _id rule.type: event.module related_ip: related.ip community_id: network.community_id event_dataset: event.dataset + hostname: host.name # Maps "opencanary" product to SO IDH logs - id: opencanary_idh_add-fields type: add_condition @@ -181,7 +145,15 @@ transformations: rule_conditions: - type: logsource category: file_event - category: file_event + # Maps network rules to all network logs + # This targets all network logs, all services, generated from endpoints and network + - id: network_add-fields + type: add_condition + conditions: + event.category: 'network' + rule_conditions: + - type: logsource + category: network # Maps network_connection rules to endpoint network creation logs # This is an OS-agnostic mapping, to account for logs that don't specify source OS - id: endpoint_network_connection_add-fields @@ -218,3 +190,41 @@ transformations: - type: logsource category: network service: dns + # Maps "network + file" to SO file logs + - id: network_file_so_add-fields + type: add_condition + conditions: + event.category: 'network' + tags: 'file' + rule_conditions: + - type: logsource + category: network + service: file + # Maps "network + x509" to SO x509 logs + - id: network_x509_so_add-fields + type: add_condition + conditions: + event.category: 'network' + tags: 'x509' + rule_conditions: + - type: logsource + category: network + service: x509 + # Maps "network + ssl" to SO ssl logs + - id: network_ssl_so_add-fields + type: add_condition + conditions: + event.category: 'network' + tags: 'ssl' + rule_conditions: + - type: logsource + category: network + service: ssl + # Maps file to host or network file events + - id: file_so_add-fields + type: add_condition + conditions: + tags: '*file' + rule_conditions: + - type: logsource + category: file \ No newline at end of file diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index d41ead0e8..1daf77102 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -57,6 +57,7 @@ zeek: - cve-2020-0601 - securityonion/bpfconf - securityonion/file-extraction + - securityonion/community-id-extended - oui-logging - icsnpp-modbus - icsnpp-dnp3 diff --git a/salt/zeek/policy/securityonion/community-id-extended.zeek b/salt/zeek/policy/securityonion/community-id-extended.zeek new file mode 100644 index 000000000..e8df10066 --- /dev/null +++ b/salt/zeek/policy/securityonion/community-id-extended.zeek @@ -0,0 +1,40 @@ +##! Extends community ID logging to Files, and SSL by copying +##! the community_id from the parent connection. +##! +##! Note: Requires that protocols/conn/community-id-logging is loaded + +module CommunityIDExt; + +@load base/protocols/ssl +@load protocols/conn/community-id-logging + +export { + redef record SSL::Info += { + community_id: string &optional &log; + }; + + redef record Files::Info += { + community_id: string &optional &log; + }; +} + +# Files +event file_new(f: fa_file) { + if ( f?$conns ) { + # Take community_id from first connection that has it + for ( cid in f$conns ) { + local c = f$conns[cid]; + if ( c?$conn && c$conn?$community_id ) { + f$info$community_id = c$conn$community_id; + break; + } + } + } +} + +# SSL Connections +event ssl_established(c: connection) { + if ( c?$conn && c$conn?$community_id && c?$ssl ) { + c$ssl$community_id = c$conn$community_id; + } +} From 58f4db95ea1505c6a0517e9fcff21093ff447595 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 May 2025 15:31:50 -0400 Subject: [PATCH 13/45] Create playbooks dir --- salt/soc/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index e19e3eb14..78a495e0a 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -52,6 +52,13 @@ socsaltdir: - mode: 770 - makedirs: True +socplaybooksdir: + file.directory: + - name: /opt/so/conf/soc/playbooks + - user: 939 + - group: 939 + - makedirs: True + socanalytics: file.managed: - name: /opt/so/conf/soc/analytics.js From 11fb33fdeb942aadebcc0a4664b130f514223628 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 19 May 2025 14:19:56 -0600 Subject: [PATCH 14/45] Add RulesetName to Rule Repos Fill in `rulesetName` in the rules repos of the ElastAlert and Strelka engines. These will act as an example to anybody adding their repos to these lists. The field is not required, but helps avoid collisions when managing repos as the value is used for the folder name. When not present, the final folder of the repo url is used as the rulesetName and as the folder name on disk. Note that rulesetNames including a `/` will create extra folders in the path but the rulesetName will contain the slash, i.e. `rulesetName="joesecurity/sigma-rules"` will create the nested structure of `reposFolder/joesecurity/sigma-rules" containing the contents of the repo. All rules imported from this repo will have the ruleset of `joesecurity/sigma-rules`. --- salt/soc/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fe190ea69..d756489e1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1415,17 +1415,21 @@ soc: license: Elastic-2.0 folder: sigma/stable community: true + rulesetName: securityonion-resources - repo: file:///nsm/rules/custom-local-repos/local-sigma license: Elastic-2.0 community: false + rulesetName: local-sigma airgap: - repo: file:///nsm/rules/detect-sigma/repos/securityonion-resources license: Elastic-2.0 folder: sigma/stable community: true + rulesetName: securityonion-resources - repo: file:///nsm/rules/custom-local-repos/local-sigma license: Elastic-2.0 community: false + rulesetName: local-sigma sigmaRulePackages: - core - emerging_threats_addon @@ -1500,16 +1504,20 @@ soc: - repo: https://github.com/Security-Onion-Solutions/securityonion-yara license: DRL community: true + rulesetName: securityonion-yara - repo: file:///nsm/rules/custom-local-repos/local-yara license: Elastic-2.0 community: false + rulesetName: local-yara airgap: - repo: file:///nsm/rules/detect-yara/repos/securityonion-yara license: DRL community: true + rulesetName: securityonion-yara - repo: file:///nsm/rules/custom-local-repos/local-yara license: Elastic-2.0 community: false + rulesetName: local-yara yaraRulesFolder: /opt/sensoroni/yara/rules stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state integrityCheckFrequencySeconds: 1200 From 39f74fe547d519a69e5b391512ccb44eba19cc31 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 19 May 2025 15:37:33 -0600 Subject: [PATCH 15/45] Use the new JSON object editor for RulesRepos config entries --- salt/soc/soc_soc.yaml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 91ab6e3c1..58560e89e 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -344,6 +344,23 @@ soc: advanced: True forcedType: "[]{}" helpLink: sigma.html + syntax: json + uiElements: + - field: rulesetName + label: Ruleset Name + - field: repo + label: Repo URL + required: True + - field: branch + label: Branch + - field: license + label: License + required: True + - field: folder + label: Folder + - field: community + label: Community + forcedType: bool airgap: *eerulesRepos sigmaRulePackages: description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, the new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' @@ -459,6 +476,23 @@ soc: advanced: True forcedType: "[]{}" helpLink: yara.html + syntax: json + uiElements: + - field: rulesetName + label: Ruleset Name + - field: repo + label: Repo URL + required: True + - field: branch + label: Branch + - field: license + label: License + required: True + - field: folder + label: Folder + - field: community + label: Community + forcedType: bool airgap: *serulesRepos suricataengine: aiRepoUrl: @@ -592,7 +626,7 @@ soc: label: Query required: True - field: showSubtitle - label: Show Query in Dropdown. + label: Show Query in Dropdown. forcedType: bool queryToggleFilters: description: Customize togglable query filters that apply to all queries. Exclusive toggles will invert the filter if toggled off rather than omitting the filter from the query. From b753d40861bb0e07bfadd18e14c6e4d40fba25d1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 20 May 2025 17:06:11 -0400 Subject: [PATCH 16/45] Tighten parsing --- salt/elasticsearch/files/ingest/global@custom | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 85154b908..5457a2703 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -24,8 +24,8 @@ { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}}, - {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}}, + {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}}, + {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint'","description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}}, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] } From ad5a27f9911aa950de71c0612dc8bf871d698dd4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 22 May 2025 13:39:59 -0400 Subject: [PATCH 17/45] clear out hf --- HOTFIX | 1 - 1 file changed, 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index bfbfc7c42..e69de29bb 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +0,0 @@ -20250522 From d3ee5ed7b8c40efa15c3eb2cb06d30262f0f40f4 Mon Sep 17 00:00:00 2001 From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com> Date: Wed, 28 May 2025 09:20:41 -0500 Subject: [PATCH 18/45] use zeek network.community_id when available --- salt/elasticsearch/files/ingest/zeek.common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index 8cf4df9be..ff62a9d5d 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -12,7 +12,8 @@ { "rename": { "field": "message2.id.orig_p", "target_field": "source.port", "ignore_missing": true } }, { "rename": { "field": "message2.id.resp_h", "target_field": "destination.ip", "ignore_missing": true } }, { "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } }, - { "community_id": {} }, + { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, + { "community_id": { "if": "ctx.network?.community_id == null" } }, { "set": { "if": "ctx.source?.ip != null", "field": "client.ip", "value": "{{source.ip}}" } }, { "set": { "if": "ctx.source?.port != null", "field": "client.port", "value": "{{source.port}}" } }, { "set": { "if": "ctx.destination?.ip != null", "field": "server.ip", "value": "{{destination.ip}}" } }, From fcdacc3b0dd838585d96e02a7f193a0f47a4f588 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 29 May 2025 12:21:28 -0500 Subject: [PATCH 19/45] fix system integration time overwrite and delete unused ingest pipeline --- salt/elasticsearch/files/ingest/global@custom | 1 - salt/elasticsearch/files/ingest/import.wel | 11 ----------- 2 files changed, 12 deletions(-) delete mode 100644 salt/elasticsearch/files/ingest/import.wel diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 5457a2703..ccf57b2d7 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -19,7 +19,6 @@ { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } }, { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel deleted file mode 100644 index d03cd34e4..000000000 --- a/salt/elasticsearch/files/ingest/import.wel +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description" : "import.wel", - "processors" : [ - { "set": { "field": "event.ingested", "value": "{{ @timestamp }}" } }, - { "set" : { "field" : "@timestamp", "value" : "{{ event.created }}" } }, - { "remove": { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } }, - { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, - { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { "pipeline": { "name": "common" } } - ] -} From 58936b31d54fd69e4aaae03bee43639a43cbba78 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 29 May 2025 15:19:21 -0400 Subject: [PATCH 20/45] FIX: Improve annotation for Elasticsearch index deletion #14682 --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 8420611f2..c268cc493 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -12,7 +12,7 @@ elasticsearch: description: Specify the memory heap size in (m)egabytes for Elasticsearch. helpLink: elasticsearch.html index_clean: - description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. + description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings. forcedType: bool helpLink: elasticsearch.html retention: From 27358137f2c788ed5232df094fd34f75598bd2a8 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 30 May 2025 09:24:41 -0400 Subject: [PATCH 21/45] FIX: so-suricata-testrule should disable pcap logging #14685 --- salt/suricata/tools/sbin_jinja/so-suricata-testrule | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/suricata/tools/sbin_jinja/so-suricata-testrule b/salt/suricata/tools/sbin_jinja/so-suricata-testrule index e1f355508..d65b630b0 100755 --- a/salt/suricata/tools/sbin_jinja/so-suricata-testrule +++ b/salt/suricata/tools/sbin_jinja/so-suricata-testrule @@ -26,7 +26,6 @@ echo "Running all.rules and $TESTRULE against the following pcap: $TESTPCAP" echo "" sleep 3 - rm -rf /tmp/nids-testing/output mkdir -p /tmp/nids-testing/output chown suricata:socore /tmp/nids-testing/output @@ -45,7 +44,7 @@ echo "==== Begin Suricata Output ===" -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \ -v /tmp/nids-testing/output/:/nsm/:rw \ {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \ - --runmode single -v -k none -r /input.pcap -l /tmp --init-errors-fatal + --runmode single -v -k none -r /input.pcap -l /tmp --init-errors-fatal --set outputs.6.pcap-log.enabled=no echo "==== End Suricata Output ===" echo "" From 45d541d4f212433f917d19e32a94538e8aae025d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 30 May 2025 09:55:53 -0400 Subject: [PATCH 22/45] FIX: so-elasticsearch-ilm-start needs shebang #14688 --- salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start index d9c63f8ea..3f1c8e41d 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start @@ -1,4 +1,4 @@ -/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the @@ -6,6 +6,5 @@ . /usr/sbin/so-common - echo "Starting ILM..." curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/start From 90b8d6b2f76b48d9a92a7fa674294ccd252b2f2e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 30 May 2025 11:41:11 -0400 Subject: [PATCH 23/45] add echo to end of so-elasticsearch-ilm-start --- salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start index 3f1c8e41d..c13d91fe5 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start @@ -8,3 +8,4 @@ echo "Starting ILM..." curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/start +echo From bf38055a6c20fb93a03bd57f99cb44e8374f962a Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 30 May 2025 11:41:50 -0400 Subject: [PATCH 24/45] add echo to end of so-elasticsearch-ilm-stop --- salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop index 034082699..e53a4939a 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop @@ -8,3 +8,4 @@ echo "Stopping ILM..." curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/stop +echo From 0277891392263ed68c6d6b95408ad9fe86ec8a5c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 2 Jun 2025 13:10:13 -0400 Subject: [PATCH 25/45] Use Stable branch --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d756489e1..f1ec6e452 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1465,9 +1465,9 @@ soc: playbookImportFrequencySeconds: 86400 playbookImportErrorSeconds: 600 playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources - playbookRepoBranch: playbook-stable + playbookRepoBranch: playbooks-stable playbookRepoPath: /opt/sensoroni/playbooks/ - playbookPathInRepo: playbook/dev + playbookPathInRepo: securityonion-normalized salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 From 1b89cc681883fab7d3b2e70154a43dae59babfd0 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 2 Jun 2025 15:41:03 -0500 Subject: [PATCH 26/45] so-elasticsearch-index-growth script --- .../tools/sbin/so-elasticsearch-index-growth | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth new file mode 100644 index 000000000..bcaa00fa2 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth @@ -0,0 +1,118 @@ +#! /bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +INFLUX_URL="https://localhost:8086/api/v2" + +. /usr/sbin/so-common + +request() { + curl -skK /opt/so/conf/influxdb/curl.config "$INFLUX_URL/$@" +} + +lookup_org_id() { + response=$(request orgs?org=Security+Onion) + echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id" +} + +ORG_ID=$(lookup_org_id) + +run_flux_query() { + local query=$1 + request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null +} + +read_csv_result() { + local result="$1" + echo "$result" | grep '^,_result,' | head -1 | awk -F',' '{print $NF}' | tr -d '\r\n\t ' +} + +bytes_to_gb() { + local bytes="${1:-0}" + if [[ "$bytes" =~ ^-?[0-9]+$ ]]; then + echo "$bytes" | awk '{printf "%.2f", $1 / 1024 / 1024 / 1024}' + else + echo "0.00" + fi +} + +indexes_query='from(bucket: "telegraf/so_long_term") +|> range(start: -7d) +|> filter(fn: (r) => r._measurement == "elasticsearch_index_size") +|> distinct(column: "_field") +|> keep(columns: ["_field"])' + +indexes_result=$(run_flux_query "$indexes_query") +indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) + +printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" +printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + +temp_results=$(mktemp) + +for index in $indexes; do + [[ -z "$index" ]] && continue + current_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -4h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + current_result=$(run_flux_query "$current_query") + current_size=$(read_csv_result "$current_result") + current_size=${current_size:-0} + + size_24h_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -25h, stop: -23h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_24h_result=$(run_flux_query "$size_24h_query") + size_24h_ago=$(read_csv_result "$size_24h_result") + size_24h_ago=${size_24h_ago:-$current_size} + + size_7d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -8d, stop: -6d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_7d_result=$(run_flux_query "$size_7d_query") + size_7d_ago=$(read_csv_result "$size_7d_result") + size_7d_ago=${size_7d_ago:-$current_size} + + size_30d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -31d, stop: -29d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_30d_result=$(run_flux_query "$size_30d_query") + size_30d_ago=$(read_csv_result "$size_30d_result") + size_30d_ago=${size_30d_ago:-$current_size} + + # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. + growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) + + growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) + + growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) + + growth_24h_gb=$(bytes_to_gb "$growth_24h") + growth_7d_gb=$(bytes_to_gb "$growth_7d") + growth_30d_gb=$(bytes_to_gb "$growth_30d") + + # Only results for indices with atleast 1 metric above 0.00 + if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" \ + "$growth_24h" \ + "$index" \ + "$growth_24h_gb" \ + "$growth_7d_gb" \ + "$growth_30d_gb" >> "$temp_results" + fi +done + +sort -t'|' -k1,1nr "$temp_results" | cut -d'|' -f2- +rm -f "$temp_results" + From 061600fa7ac3d9978945a9aa655d48c1eae84dee Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 2 Jun 2025 15:55:46 -0500 Subject: [PATCH 27/45] shebang line --- salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth index bcaa00fa2..801f64342 100644 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at From 6238a5b3edf048dfa5ba70e0948bb69262ffb4ff Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 2 Jun 2025 16:31:26 -0500 Subject: [PATCH 28/45] tighten up search timeframe --- salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth index 801f64342..7e31a5eed 100644 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth @@ -74,7 +74,7 @@ for index in $indexes; do size_24h_ago=${size_24h_ago:-$current_size} size_7d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -8d, stop: -6d) + |> range(start: -7d8h, stop: -7d) |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") |> last() |> keep(columns: [\"_value\"])" @@ -83,7 +83,7 @@ for index in $indexes; do size_7d_ago=${size_7d_ago:-$current_size} size_30d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -31d, stop: -29d) + |> range(start: -30d8h, stop: -30d) |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") |> last() |> keep(columns: [\"_value\"])" From 4d6171bde6aeac8bdf1d7e78d2d3580e030fcca7 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 3 Jun 2025 07:32:12 -0500 Subject: [PATCH 29/45] rename script Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- ...elasticsearch-index-growth => so-elasticsearch-indices-growth} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/elasticsearch/tools/sbin/{so-elasticsearch-index-growth => so-elasticsearch-indices-growth} (100%) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth similarity index 100% rename from salt/elasticsearch/tools/sbin/so-elasticsearch-index-growth rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth From d240fca72108b4e8af2d11454cf83567665a6326 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:45:04 -0500 Subject: [PATCH 30/45] remove usage of temp file --- .../tools/sbin/so-elasticsearch-indices-growth | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth index 7e31a5eed..3381947eb 100644 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth @@ -51,8 +51,6 @@ indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | g printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" -temp_results=$(mktemp) - for index in $indexes; do [[ -z "$index" ]] && continue current_query="from(bucket: \"telegraf/so_long_term\") @@ -109,10 +107,7 @@ for index in $indexes; do "$index" \ "$growth_24h_gb" \ "$growth_7d_gb" \ - "$growth_30d_gb" >> "$temp_results" + "$growth_30d_gb" fi -done - -sort -t'|' -k1,1nr "$temp_results" | cut -d'|' -f2- -rm -f "$temp_results" +done | sort -t'|' -k1,1nr | cut -d'|' -f2- From fc9107f1290334fd12fbda3b637cd695c8c22c3e Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 3 Jun 2025 13:33:30 -0600 Subject: [PATCH 31/45] Updated Playbook Repo Config The repo and folder have changed. We're splitting out playbooks into their own repo: github.com/security-onion-solutions/securityonion-resources-playbooks. --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index f1ec6e452..24b9aaba2 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1464,8 +1464,8 @@ soc: autoUpdateEnabled: true playbookImportFrequencySeconds: 86400 playbookImportErrorSeconds: 600 - playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources - playbookRepoBranch: playbooks-stable + playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + playbookRepoBranch: main playbookRepoPath: /opt/sensoroni/playbooks/ playbookPathInRepo: securityonion-normalized salt: From 832d66052e92d9de36a3494bc486dc8e7bdc5c99 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 4 Jun 2025 09:13:54 -0400 Subject: [PATCH 32/45] upgrade registry to 3.0.0 --- salt/registry/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index bff46dd5b..592afd976 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -14,7 +14,7 @@ include: # Install the registry container so-dockerregistry: docker_container.running: - - image: ghcr.io/security-onion-solutions/registry:2.8.3 + - image: ghcr.io/security-onion-solutions/registry:3.0.0 - hostname: so-registry - networks: - sobridge: From ab89858d04f61809f8198774c52dca6787c3d5e7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 4 Jun 2025 17:19:53 -0400 Subject: [PATCH 33/45] update to new config location --- salt/registry/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index 592afd976..88eea6dc3 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -25,7 +25,7 @@ so-dockerregistry: - {{ BINDING }} {% endfor %} - binds: - - /opt/so/conf/docker-registry/etc/config.yml:/etc/docker/registry/config.yml:ro + - /opt/so/conf/docker-registry/etc/config.yml:/etc/distribution/config.yml:ro - /opt/so/conf/docker-registry:/var/lib/registry:rw - /nsm/docker-registry/docker:/var/lib/registry/docker:rw - /etc/pki/registry.crt:/etc/pki/registry.crt:ro From 643afeeae75fd5c3c67cb6e461b2969548c31282 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 5 Jun 2025 16:02:27 -0400 Subject: [PATCH 34/45] enable STS for browser redirects --- salt/nginx/etc/nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 069e55cdb..5fa98a882 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -383,6 +383,7 @@ http { } if ($request_uri ~* ^/(?!(^/api/.*))) { add_header Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400"; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; } return 302 /auth/self-service/login/browser; } @@ -392,6 +393,7 @@ http { return 403; } add_header Set-Cookie "ory_kratos_session=;Path=/;Max-Age=0;expires=Thu, 01 Jan 1970 00:00:00 GMT;"; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; return 302 /auth/self-service/login/browser; } From a3b5db5945fd2a8a315f17aed3aa8fe8e62b9e7a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 6 Jun 2025 16:17:14 -0400 Subject: [PATCH 35/45] Add support for Airgap for Playbooks --- salt/manager/tools/sbin/soup | 8 ++++++++ salt/soc/defaults.yaml | 4 +++- salt/soc/merged.map.jinja | 7 +++++++ setup/so-functions | 6 ++++++ setup/so-setup | 4 ++++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 987c599f1..257699c4e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1069,6 +1069,11 @@ update_airgap_rules() { rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/ } +update_airgap_playbooks() { + # Copy the playbooks over to update them for airgap. + rsync -a --delete --chown=socore:socore $UPDATE_DIR/airgap-resources/playbooks/* /nsm/airgap-resources/playbooks/ +} + update_airgap_repo() { # Update the files in the repo echo "Syncing new updates to /nsm/repo" @@ -1253,6 +1258,7 @@ apply_hotfix() { elif [[ "$INSTALLEDVERSION" == "2.4.30" ]] ; then if [[ $is_airgap -eq 0 ]]; then update_airgap_rules + update_airgap_playbooks fi if [[ -f /etc/pki/managerssl.key.old ]]; then echo "Skipping Certificate Generation" @@ -1418,6 +1424,8 @@ main() { if [[ $is_airgap -eq 0 ]]; then echo "Updating Rule Files to the Latest." update_airgap_rules + echo "Updating Playbooks to the Latest." + update_airgap_playbooks fi # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 24b9aaba2..66355fa24 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1464,7 +1464,9 @@ soc: autoUpdateEnabled: true playbookImportFrequencySeconds: 86400 playbookImportErrorSeconds: 600 - playbookRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + playbookRepoUrl: + default: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + airgap: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks playbookRepoBranch: main playbookRepoPath: /opt/sensoroni/playbooks/ playbookPathInRepo: securityonion-normalized diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 25b62683c..c950d8a60 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -61,6 +61,13 @@ {% do SOCMERGED.config.server.update({'airgapEnabled': false}) %} {% endif %} +{# set playbookRepoUrl based on airgap or not #} +{% if GLOBALS.airgap %} +{% do SOCMERGED.config.server.modules.playbook.update({'playbookRepoUrl': SOCMERGED.config.server.modules.playbook.playbookRepoUrl.airgap}) %} +{% else %} +{% do SOCMERGED.config.server.modules.playbook.update({'playbookRepoUrl': SOCMERGED.config.server.modules.playbook.playbookRepoUrl.default}) %} +{% endif %} + {# remove these modules if detections is disabled #} {% if not SOCMERGED.config.server.client.detectionsEnabled %} {% do SOCMERGED.config.server.modules.pop('elastalertengine') %} diff --git a/setup/so-functions b/setup/so-functions index 82dd9d375..54fb49f9f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -56,6 +56,12 @@ airgap_detection_summaries() { logCmd "git config --global --add safe.directory /opt/so/conf/soc/ai_summary_repos/securityonion-resources" logCmd "git -C /opt/so/conf/soc/ai_summary_repos/securityonion-resources checkout generated-summaries-published" } +airgap_playbooks() { + # Copy playbooks if using airgap + mkdir -p /nsm/airgap-resources/playbooks + logCmd "rsync -av --chown=socore:socore /root/SecurityOnion/airgap-resources/playbooks/ /nsm/airgap-resources/playbooks/" + logCmd "git config --global --add safe.directory /nsm/airgap-resources/playbooks" +} add_admin_user() { title "Adding $ADMINUSER to the system with sudo rights" diff --git a/setup/so-setup b/setup/so-setup index b7723da37..7f2e87c16 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -769,6 +769,10 @@ if ! [[ -f $install_opt_file ]]; then title "Syncing AI-Generated Detection Summaries" airgap_detection_summaries fi + if [[ $is_airgap ]]; then + title "Syncing Playbooks" + airgap_playbooks + fi title "Setting up Kibana Default Space" logCmd "so-kibana-space-defaults" add_web_user From b428573a0ad7a6916f20718f15b26aad2b44eea4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 6 Jun 2025 17:48:49 -0400 Subject: [PATCH 36/45] Airgap tweaks --- salt/manager/init.sls | 9 +++++++++ salt/manager/tools/sbin/soup | 3 +-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 07a1b8816..737d753f4 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -156,6 +156,13 @@ rules_dir: - group: socore - makedirs: True +nsm_playbooks_dir: + file.directory: + - name: /nsm/airgap-resources/playbooks + - user: socore + - group: socore + - makedirs: True + git_config_set_safe_dirs: git.config_set: - name: safe.directory @@ -166,6 +173,8 @@ git_config_set_safe_dirs: - /nsm/rules/custom-local-repos/local-yara - /nsm/securityonion-resources - /opt/so/conf/soc/ai_summary_repos/securityonion-resources + - /nsm/airgap-resources/playbooks + - /opt/so/conf/soc/playbooks {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 257699c4e..d1c84ec2f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1071,7 +1071,7 @@ update_airgap_rules() { update_airgap_playbooks() { # Copy the playbooks over to update them for airgap. - rsync -a --delete --chown=socore:socore $UPDATE_DIR/airgap-resources/playbooks/* /nsm/airgap-resources/playbooks/ + rsync -a --delete --chown=socore:socore $UPDATE_DIR/airgap-resources/playbooks/ /nsm/airgap-resources/playbooks/ } update_airgap_repo() { @@ -1258,7 +1258,6 @@ apply_hotfix() { elif [[ "$INSTALLEDVERSION" == "2.4.30" ]] ; then if [[ $is_airgap -eq 0 ]]; then update_airgap_rules - update_airgap_playbooks fi if [[ -f /etc/pki/managerssl.key.old ]]; then echo "Skipping Certificate Generation" From f360c6ecbc5f07302cccf279b3e0415d5e77c74a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 7 Jun 2025 09:29:59 -0400 Subject: [PATCH 37/45] Supress alerts --- salt/idstools/rules/extraction.rules | 40 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/salt/idstools/rules/extraction.rules b/salt/idstools/rules/extraction.rules index bccfc69d6..3ebbd41b1 100644 --- a/salt/idstools/rules/extraction.rules +++ b/salt/idstools/rules/extraction.rules @@ -1,26 +1,26 @@ # Extract all PDF mime type -alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100000; rev:1;) -alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100001; rev:1;) -alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100002; rev:1;) -alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100003; rev:1;) +alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;) +alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;) +alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;) +alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;) # Extract EXE/DLL file types -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100007; rev:1;) -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100011; rev:1;) +alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;) +alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;) +alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;) +alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;) +alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;) +alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;) +alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;) +alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;) # Extract all Zip files -alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;) -alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;) -alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;) -alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100015; rev:1;) +alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;) +alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;) +alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;) +alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;) # Extract Word Docs -alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;) -alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;) -alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;) -alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;) \ No newline at end of file +alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;) +alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;) +alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;) +alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;) \ No newline at end of file From dbdbffa4b01a4ba87a2b9c8a145c0a21fa3e386a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 8 Jun 2025 08:23:09 -0400 Subject: [PATCH 38/45] Add nsm bind --- salt/soc/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 4b39d036e..09e2c16a8 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -31,6 +31,7 @@ so-soc: - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw + - /nsm/airgap-resources:/nsm/airgap-resources:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/ai_summary_repos:/opt/sensoroni/ai_summary_repos:rw From a600c64229c83fd454c56aac255dcb39e6b022a7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 9 Jun 2025 07:33:02 -0400 Subject: [PATCH 39/45] Create dir if needed --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d1c84ec2f..aa030bcdb 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1071,6 +1071,8 @@ update_airgap_rules() { update_airgap_playbooks() { # Copy the playbooks over to update them for airgap. + mkdir -p /nsm/airgap-resources/playbooks + chown -R socore:socore /nsm/airgap-resources/playbooks rsync -a --delete --chown=socore:socore $UPDATE_DIR/airgap-resources/playbooks/ /nsm/airgap-resources/playbooks/ } From 31cd5b136503ffc2c1a5769e7c2bb35285098faf Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 20 Jun 2025 15:02:59 -0400 Subject: [PATCH 40/45] Add support for dns.resolved_ip --- salt/elasticsearch/files/ingest/zeek.dns | 5 +++-- salt/soc/files/soc/sigma_so_pipeline.yaml | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 57a9347a5..5616310bd 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -20,7 +20,8 @@ { "rename": { "field": "message2.RD", "target_field": "dns.recursion.desired", "ignore_missing": true } }, { "rename": { "field": "message2.RA", "target_field": "dns.recursion.available", "ignore_missing": true } }, { "rename": { "field": "message2.Z", "target_field": "dns.reserved", "ignore_missing": true } }, - { "rename": { "field": "message2.answers", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "rename": { "field": "message2.answers", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "script": { "lang": "painless", "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null", "source": "def ips = []; for (item in ctx.dns.answers.name) { if (item =~ /^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/ || item =~ /^([a-fA-F0-9:]+:+)+[a-fA-F0-9]+$/) { ips.add(item); } } ctx.dns.resolved_ip = ips;" } }, { "rename": { "field": "message2.TTLs", "target_field": "dns.ttls", "ignore_missing": true } }, { "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } }, @@ -28,4 +29,4 @@ { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "zeek.common" } } ] -} +} \ No newline at end of file diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 5cee0cfd3..4462bde42 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -190,6 +190,15 @@ transformations: - type: logsource category: network service: dns + # Maps "network + http" to SO HTTP logs + - id: network_http_so_add-fields + type: add_condition + conditions: + tags: 'http' + rule_conditions: + - type: logsource + category: network + service: http # Maps "network + file" to SO file logs - id: network_file_so_add-fields type: add_condition From b052a75e641576d81fdbb13422f0df032aaffb83 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jun 2025 09:34:57 -0400 Subject: [PATCH 41/45] refactor airgap playbook to eliminate dupe code and shrink ISO --- salt/common/tools/sbin/so-common | 11 +++++++++++ salt/manager/tools/sbin/soup | 9 +-------- setup/so-functions | 6 ------ setup/so-setup | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 3e05614d6..0fe46d7bb 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -99,6 +99,17 @@ add_interface_bond0() { fi } +airgap_playbooks() { + SRC_DIR=$1 + # Copy playbooks if using airgap + mkdir -p /nsm/airgap-resources + # Purge old airgap playbooks to ensure SO only uses the latest released playbooks + rm -fr /nsm/airgap-resources/playbooks + logCmd "tar xf $SRC_DIR/airgap-resources/playbooks.tgz -C /nsm/airgap-resources/" + logCmd "chown -R socore:socore /nsm/airgap-resources/playbooks" + logCmd "git config --global --add safe.directory /nsm/airgap-resources/playbooks" +} + check_container() { docker ps | grep "$1:" > /dev/null 2>&1 return $? diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index aa030bcdb..d27239228 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1069,13 +1069,6 @@ update_airgap_rules() { rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/ } -update_airgap_playbooks() { - # Copy the playbooks over to update them for airgap. - mkdir -p /nsm/airgap-resources/playbooks - chown -R socore:socore /nsm/airgap-resources/playbooks - rsync -a --delete --chown=socore:socore $UPDATE_DIR/airgap-resources/playbooks/ /nsm/airgap-resources/playbooks/ -} - update_airgap_repo() { # Update the files in the repo echo "Syncing new updates to /nsm/repo" @@ -1426,7 +1419,7 @@ main() { echo "Updating Rule Files to the Latest." update_airgap_rules echo "Updating Playbooks to the Latest." - update_airgap_playbooks + airgap_playbooks "$UPDATE_DIR" fi # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars diff --git a/setup/so-functions b/setup/so-functions index 54fb49f9f..82dd9d375 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -56,12 +56,6 @@ airgap_detection_summaries() { logCmd "git config --global --add safe.directory /opt/so/conf/soc/ai_summary_repos/securityonion-resources" logCmd "git -C /opt/so/conf/soc/ai_summary_repos/securityonion-resources checkout generated-summaries-published" } -airgap_playbooks() { - # Copy playbooks if using airgap - mkdir -p /nsm/airgap-resources/playbooks - logCmd "rsync -av --chown=socore:socore /root/SecurityOnion/airgap-resources/playbooks/ /nsm/airgap-resources/playbooks/" - logCmd "git config --global --add safe.directory /nsm/airgap-resources/playbooks" -} add_admin_user() { title "Adding $ADMINUSER to the system with sudo rights" diff --git a/setup/so-setup b/setup/so-setup index 7f2e87c16..784d9d623 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -771,7 +771,7 @@ if ! [[ -f $install_opt_file ]]; then fi if [[ $is_airgap ]]; then title "Syncing Playbooks" - airgap_playbooks + airgap_playbooks /root/SecurityOnion fi title "Setting up Kibana Default Space" logCmd "so-kibana-space-defaults" From 21d9964827efc30a191f99137c3ffc6cf1261770 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jun 2025 11:03:08 -0400 Subject: [PATCH 42/45] fix logging --- salt/common/tools/sbin/so-common | 6 +++--- setup/so-setup | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0fe46d7bb..203b54cd0 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -105,9 +105,9 @@ airgap_playbooks() { mkdir -p /nsm/airgap-resources # Purge old airgap playbooks to ensure SO only uses the latest released playbooks rm -fr /nsm/airgap-resources/playbooks - logCmd "tar xf $SRC_DIR/airgap-resources/playbooks.tgz -C /nsm/airgap-resources/" - logCmd "chown -R socore:socore /nsm/airgap-resources/playbooks" - logCmd "git config --global --add safe.directory /nsm/airgap-resources/playbooks" + tar xf $SRC_DIR/airgap-resources/playbooks.tgz -C /nsm/airgap-resources/ + chown -R socore:socore /nsm/airgap-resources/playbooks + git config --global --add safe.directory /nsm/airgap-resources/playbooks } check_container() { diff --git a/setup/so-setup b/setup/so-setup index 784d9d623..bb3779035 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -771,7 +771,7 @@ if ! [[ -f $install_opt_file ]]; then fi if [[ $is_airgap ]]; then title "Syncing Playbooks" - airgap_playbooks /root/SecurityOnion + logCmd "airgap_playbooks /root/SecurityOnion" fi title "Setting up Kibana Default Space" logCmd "so-kibana-space-defaults" From 38c74b46b613662b03d926ece6797faaa60b6005 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Jun 2025 11:05:28 -0400 Subject: [PATCH 43/45] change salt upgrade process --- salt/salt/minion.sls | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 9f2a6f3fa..57bb27694 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -39,11 +39,17 @@ unhold_salt_packages: install_salt_minion: cmd.run: - - name: | - exec 0>&- # close stdin - exec 1>&- # close stdout - exec 2>&- # close stderr - nohup /bin/sh -c '{{ UPGRADECOMMAND }}' & + - name: /bin/sh -c '{{ UPGRADECOMMAND }}' + +# minion service is in failed state after upgrade. this command will start it after the state run for the upgrade completes +start_minion: + cmd.run: + - name: nohup /bin/sh -c 'sleep 30; systemctl start salt-minion' & + - require: + - cmd: install_salt_minion + - watch: + - cmd: install_salt_minion + - order: last {% endif %} From bc24a6c574065b38656a5869b4be65d9eeb77551 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Jun 2025 12:19:45 -0400 Subject: [PATCH 44/45] Revert "change salt upgrade process" --- salt/salt/minion.sls | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 57bb27694..9f2a6f3fa 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -39,17 +39,11 @@ unhold_salt_packages: install_salt_minion: cmd.run: - - name: /bin/sh -c '{{ UPGRADECOMMAND }}' - -# minion service is in failed state after upgrade. this command will start it after the state run for the upgrade completes -start_minion: - cmd.run: - - name: nohup /bin/sh -c 'sleep 30; systemctl start salt-minion' & - - require: - - cmd: install_salt_minion - - watch: - - cmd: install_salt_minion - - order: last + - name: | + exec 0>&- # close stdin + exec 1>&- # close stdout + exec 2>&- # close stderr + nohup /bin/sh -c '{{ UPGRADECOMMAND }}' & {% endif %} From 7e746b87c5a822fca75e737ce99067cbe3a029c7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Jun 2025 13:00:26 -0400 Subject: [PATCH 45/45] 2.4.160 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++---------- sigs/securityonion-2.4.160-20250625.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.160-20250625.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 0e726eaeb..2e0b2aa5d 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.150-20250522 ISO image released on 2025/05/22 +### 2.4.160-20250625 ISO image released on 2025/06/25 ### Download and Verify -2.4.150-20250522 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250522.iso +2.4.160-20250625 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.160-20250625.iso -MD5: 239E69B83072BBF2602D4043FE53A160 -SHA1: C62893D3C7F5592665BFDCBC9A45BB20A926F9A8 -SHA256: 2ADE037C7FD34591030B1FAC10392C4E6613F152DD24BFBD897E57EE300895B9 +MD5: 78CF5602EFFAB84174C56AD2826E6E4E +SHA1: FC7EEC3EC95D97D3337501BAA7CA8CAE7C0E15EA +SHA256: 0ED965E8BEC80EE16AE90A0F0F96A3046CEF2D92720A587278DDDE3B656C01C2 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250522.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.160-20250625.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250522.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.160-20250625.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250522.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.160-20250625.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.150-20250522.iso.sig securityonion-2.4.150-20250522.iso +gpg --verify securityonion-2.4.160-20250625.iso.sig securityonion-2.4.160-20250625.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Thu 22 May 2025 11:15:06 AM EDT using RSA key ID FE507013 +gpg: Signature made Wed 25 Jun 2025 10:13:33 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.160-20250625.iso.sig b/sigs/securityonion-2.4.160-20250625.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..0145af053164dc7b1cbd40b9a2978a1140aaeb66 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%%+W4FC!W5PT3| zxBgIY6Cy|t|5BF?^=?}sFeF9#Kk!gQ5#*(!ZvGydNg&f^^0EsGaXs z^N^s0KDu5#@4cMe1Ajik?ax+|5ZCmeU&di=c&@L^yy6wp3!yLk{K8iFi-Qb7gTmK< zGB45)?0)bJ8o#EE;|)w}yksi%0{=T(!PB4eD$)BzZz6M$n|&z|;llTXY!FQSsMeoO zN7uo-MU!fzFdMDexJSjG{v;Qa8Bc3gX4A)8Sovj0OGWxqqaHGyH6NKoqlAIpmlOT9 zPRc?~se3-Mmg6NQB*+#Fs5_6ogK0?hN~4g=KlL*jj%KLgTlf#B%rHk>^mVp0Qp*%! z8#{gov}O=#gx0;7MyGu4_fXvpCAT7O4mq?IP$<$y#GF+yWL^Cx8v$(EqY#zw_;4`L zE>Q^IU~~?Yd2jM5A~!2iCbJW8I0#)wU39OdvhMaR1 z&*LZa*C!(IEvJRUhf88o6BzkJ{ANqtZ^qAdwHQojD=i)^vcq>0D2Qd&kB_E};8;heM4M)OW7@fqMM~-;}wBSv>UwC9Z EkVcdVQvd(} literal 0 HcmV?d00001