From 2df9715dc1d7349545a57b5579338d6f81a579f4 Mon Sep 17 00:00:00 2001 From: Shirofune-Security <43838376+Shirofune-Security@users.noreply.github.com> Date: Sat, 19 Sep 2026 07:39:47 +0900 Subject: [PATCH] Use explicit canonical framing for portable metadata hashes --- docs/native-rule-eligibility.md | 2 +- modules/RuleEligibility.psm1 | 58 ++++++++++++++++++++++++++++----- tests/RuleEligibility.Tests.ps1 | 16 ++++++++- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/docs/native-rule-eligibility.md b/docs/native-rule-eligibility.md index 5d3c21f2..6c0c96a0 100644 --- a/docs/native-rule-eligibility.md +++ b/docs/native-rule-eligibility.md @@ -22,7 +22,7 @@ The manifest records the exact corpus and EventID-mapping SHA256 values and coun | NotApplicable | All unambiguous, canonical Security event sources belong to other roles than the explicitly selected role. | | Excluded | An explicit Sysmon or identified external-product source. Its ID and exclusion reason remain in the output. | -Every unique rule has reasons and a metadata hash. `ordered-json-html-escaped-utf8-sha256-v1` hashes compact JSON with the fixed field order `id,title,level,category,service,channel,event_ids,subcategory_guids,description,tags`, explicit HTML escaping and UTF-8 without a BOM; this keeps per-rule identities stable across PowerShell 5.1 and 7. Corpus, mapping and imported-artifact hashes always cover the exact file bytes. Git attributes preserve LF checkouts for the shipped pinned inputs, without normalizing imported evidence. `PolicyPrerequisites` inventories the catalog's requirements; it is not a separate failed-check verdict. Identical duplicate IDs count once; conflicting duplicate IDs are rejected. The report provides input record count, unique rule count, native candidate count, role-applicable count and exclusion groups. It reports **Ready / native candidates**, **Ready / applicable candidates**, and **Ready / full unique corpus** separately. Empty denominators produce null percentages. Unknown and incomplete native candidates remain in the denominator, including generic categories lacking a verified adapter. These denominators therefore differ from the earlier standalone comparison report's explicitly narrower modeling boundary; do not compare their percentages without reconciling scope and corpus versions. +Every unique rule has reasons and a metadata hash. `wela-metadata-framed-utf16le-base64-sha256-v1` uses an explicit ASCII framing independent of JSON serialization. It begins with `wela-metadata-framed-v1;`, followed by each name and value in the fixed field order `id,title,level,category,service,channel,event_ids,subcategory_guids,description,tags`. Strings are `s:;`, with no Unicode normalization or replacement of isolated surrogates. Null is `n;`, booleans `b0;`/`b1;`, integers `i;`, floats `f<16 lowercase hexadecimal IEEE-754 double bits>;`, and decimals `m;`. Arrays are `a:;`; objects are `o:;`. The resulting ASCII bytes are SHA256-hashed; UTF-8 without a BOM encodes those bytes identically. Type tags, counts and Base64 framing distinguish null, empty values, strings, numbers and array boundaries across PowerShell 5.1 and 7. Corpus, mapping and imported-artifact hashes always cover the exact file bytes. Git attributes preserve LF checkouts for the shipped pinned inputs, without normalizing imported evidence. `PolicyPrerequisites` inventories the catalog's requirements; it is not a separate failed-check verdict. Identical duplicate IDs count once; conflicting duplicate IDs are rejected. The report provides input record count, unique rule count, native candidate count, role-applicable count and exclusion groups. It reports **Ready / native candidates**, **Ready / applicable candidates**, and **Ready / full unique corpus** separately. Empty denominators produce null percentages. Unknown and incomplete native candidates remain in the denominator, including generic categories lacking a verified adapter. These denominators therefore differ from the earlier standalone comparison report's explicitly narrower modeling boundary; do not compare their percentages without reconciling scope and corpus versions. Category-only GUIDs, blank EventID rows and ambiguous mappings cannot establish subcategory/outcome readiness. In particular, the mapping lists both Token Right Adjusted Events and Authorization Policy Change for 4703; the importer does not arbitrarily choose one. Directory-service and certificate-template change events must be assessed on their source DC, not credited automatically to a member-server CA. diff --git a/modules/RuleEligibility.psm1 b/modules/RuleEligibility.psm1 index fb9f213e..91fa5d94 100644 --- a/modules/RuleEligibility.psm1 +++ b/modules/RuleEligibility.psm1 @@ -51,17 +51,59 @@ function Read-WelaEligibilityInput { [pscustomobject]@{ Sha256=$sha256; Text=[Text.Encoding]::UTF8.GetString($bytes).TrimStart([char]0xFEFF) } } +function ConvertTo-WelaEligibilityCanonicalValue { + param($Value, [int]$Depth = 0) + if ($Depth -gt 12) { throw 'Metadata nesting exceeds 12 levels.' } + if ($null -eq $Value) { return 'n;' } + if ($Value -is [bool]) { if ($Value) { return 'b1;' }; return 'b0;' } + $invariant = [Globalization.CultureInfo]::InvariantCulture + if ($Value -is [string]) { + # Copy exact UTF-16 code units, including isolated surrogates. Encoding + # fallback must never turn distinct strings into the same identity. + $bytes = New-Object byte[] ($Value.Length * 2) + if ($bytes.Length) { [Buffer]::BlockCopy($Value.ToCharArray(), 0, $bytes, 0, $bytes.Length) } + if (-not [BitConverter]::IsLittleEndian) { + for ($i = 0; $i -lt $bytes.Length; $i += 2) { $first=$bytes[$i]; $bytes[$i]=$bytes[$i+1]; $bytes[$i+1]=$first } + } + return ('s' + $Value.Length.ToString($invariant) + ':' + [Convert]::ToBase64String($bytes) + ';') + } + if ($Value -is [byte] -or $Value -is [sbyte] -or $Value -is [int16] -or $Value -is [uint16] -or + $Value -is [int32] -or $Value -is [uint32] -or $Value -is [int64] -or $Value -is [uint64]) { + return ('i' + $Value.ToString($invariant) + ';') + } + if ($Value -is [single] -or $Value -is [double]) { + return ('f' + [BitConverter]::DoubleToInt64Bits([double]$Value).ToString('x16', $invariant) + ';') + } + if ($Value -is [decimal]) { return ('m' + $Value.ToString('G29', $invariant) + ';') } + $text = New-Object Text.StringBuilder + if ($Value -is [array]) { + [void]$text.Append('a' + $Value.Count.ToString($invariant) + ':') + foreach ($item in $Value) { [void]$text.Append((ConvertTo-WelaEligibilityCanonicalValue -Value $item -Depth ($Depth + 1))) } + } elseif ($Value -is [Collections.IDictionary] -or $Value -is [pscustomobject]) { + $dictionary = $Value -is [Collections.IDictionary] + [string[]]$names = @() + if ($dictionary) { $names = @($Value.Keys) } else { $names = @($Value.PSObject.Properties | ForEach-Object { $_.Name }) } + [Array]::Sort($names, [StringComparer]::Ordinal) + [void]$text.Append('o' + $names.Count.ToString($invariant) + ':') + foreach ($name in $names) { + [void]$text.Append((ConvertTo-WelaEligibilityCanonicalValue -Value $name -Depth ($Depth + 1))) + if ($dictionary) { $member = $Value[$name] } else { $member = $Value.$name } + [void]$text.Append((ConvertTo-WelaEligibilityCanonicalValue -Value $member -Depth ($Depth + 1))) + } + } else { throw ('Unsupported metadata value type: ' + $Value.GetType().FullName) } + [void]$text.Append(';') + return $text.ToString() +} + function Get-WelaEligibilityRuleHash { param($Rule) - $ordered = [ordered]@{} + $text = New-Object Text.StringBuilder + [void]$text.Append('wela-metadata-framed-v1;') foreach ($name in @('id', 'title', 'level', 'category', 'service', 'channel', 'event_ids', 'subcategory_guids', 'description', 'tags')) { - $ordered[$name] = $Rule.$name + [void]$text.Append((ConvertTo-WelaEligibilityCanonicalValue -Value $name)) + [void]$text.Append((ConvertTo-WelaEligibilityCanonicalValue -Value $Rule.$name)) } - # Windows PowerShell 5.1 uses HTML escaping by default. Require the same - # spelling on newer editions so identical metadata has one stable digest. - $arguments = @{InputObject=$ordered;Depth=12;Compress=$true} - if ((Get-Command ConvertTo-Json).Parameters.ContainsKey('EscapeHandling')) { $arguments.EscapeHandling = 'EscapeHtml' } - Get-WelaEligibilityTextHash (ConvertTo-Json @arguments) + Get-WelaEligibilityTextHash $text.ToString() } function Get-WelaEligibilityArtifact { @@ -366,7 +408,7 @@ function Get-WelaRuleEligibility { [pscustomobject][ordered]@{ SchemaVersion = 1; GeneratedAtUtc = $Now.ToString('o'); Scope = 'native-windows-rule-eligibility' AssessmentBasis = $(if ($EvidencePath) { 'Imported lab artifacts; Ready applies only to the recorded context/time and is not a current-host or universal guarantee.' } else { 'Metadata/configuration assessment only; no event-generation, ingestion or query evidence imported.' }) - Corpus = [pscustomobject]@{ Sha256 = $corpusHash; MappingSha256 = $mappingHash; Pinned = [bool]$pinned; Manifest = $manifest; MetadataHashAlgorithm = 'ordered-json-html-escaped-utf8-sha256-v1'; Kind = 'WELA extracted Hayabusa rule metadata; not the complete upstream Sigma corpus' } + Corpus = [pscustomobject]@{ Sha256 = $corpusHash; MappingSha256 = $mappingHash; Pinned = [bool]$pinned; Manifest = $manifest; MetadataHashAlgorithm = 'wela-metadata-framed-utf16le-base64-sha256-v1'; Kind = 'WELA extracted Hayabusa rule metadata; not the complete upstream Sigma corpus' } RequestedContext = [pscustomobject]@{ Role = $Role; Build = $(if ($Build) { $Build } else { $null }) } Summary = [pscustomobject]@{ InputRecords = $raw.Count; UniqueRules = $rows.Count; DuplicateRecords = $duplicateCount diff --git a/tests/RuleEligibility.Tests.ps1 b/tests/RuleEligibility.Tests.ps1 index a64dc570..5e621aa7 100644 --- a/tests/RuleEligibility.Tests.ps1 +++ b/tests/RuleEligibility.Tests.ps1 @@ -59,7 +59,21 @@ try { $hashVector.title='Apostrophe' + [char]0x27 + 's & "quoted" \ path ' + [char]0x65e5 + [char]0x672c + [char]0x8a9e + [char]::ConvertFromUtf32(0x1f600) $hashVector.description="line`nnext`ttab" $actualMetadataHash=& (Get-Module RuleEligibility) { param($rule) Get-WelaEligibilityRuleHash $rule } $hashVector - Assert ($actualMetadataHash -eq '8d81c215ae99b303fd30e346613a35b83a1949d7cfa4f436952ef3079b8d0ded') ("Metadata hash is independent of PowerShell JSON escaping: " + $actualMetadataHash) + # Expected digest independently calculated from the documented framing in Python. + Assert ($actualMetadataHash -eq 'cfa5e1a0f73d8f0eef62716a44e56334461bbf55ddf0a07b39ee63e378597e9d') ("Metadata hash is independent of PowerShell JSON serialization: " + $actualMetadataHash) + function Canonical($Value) { & (Get-Module RuleEligibility) { param($item) ConvertTo-WelaEligibilityCanonicalValue -Value $item } $Value } + foreach ($pair in @( + [pscustomobject]@{Left=$null;Right=''}, [pscustomobject]@{Left=$null;Right=@()}, + [pscustomobject]@{Left=@();Right=[pscustomobject]@{}}, [pscustomobject]@{Left=1;Right='1'}, + [pscustomobject]@{Left=$true;Right=1}, [pscustomobject]@{Left=1;Right=1.0}, + [pscustomobject]@{Left=@('ab','c');Right=@('a','bc')}, [pscustomobject]@{Left='a';Right=@('a')}, + [pscustomobject]@{Left=@('a','b');Right=@('b','a')}, + [pscustomobject]@{Left=([string][char]0xd800);Right=([string][char]0xfffd)}, + [pscustomobject]@{Left=([string][char]0x00e9);Right=('e'+[char]0x0301)} + )) { Assert ((Canonical $pair.Left) -cne (Canonical $pair.Right)) 'Canonical framing preserves type, boundaries, array order and exact Unicode code units.' } + Assert ((Canonical ([pscustomobject][ordered]@{z=@('x');a=@()})) -ceq (Canonical ([pscustomobject][ordered]@{a=@();z=@('x')}))) 'Object identity uses ordinal property order rather than insertion order.' + Assert ((Canonical ([pscustomobject]@{a=@('x')})) -cne (Canonical ([pscustomobject]@{a='x'}))) 'Nested singleton arrays retain their type.' + Assert ((Canonical ([pscustomobject]@{a=@()})) -cne (Canonical ([pscustomobject]@{a=$null}))) 'Nested empty arrays never collapse into null.' Save-Corpus @(Fixture-Rule) $r=Report Assert ($r.Summary.Ready -eq 0 -and $r.Results[0].State -eq 'Conditional') 'Lossy metadata cannot demonstrate usable rules.'