From e8a0aeb59bb3baed4462e07945a87f88c1f203ef Mon Sep 17 00:00:00 2001 From: Shirofune-Security <43838376+Shirofune-Security@users.noreply.github.com> Date: Sat, 19 Sep 2026 07:34:10 +0900 Subject: [PATCH] Keep rule evidence identities stable across Windows checkouts and shells --- .gitattributes | 4 ++++ docs/native-rule-eligibility.md | 2 +- modules/RuleEligibility.psm1 | 8 ++++++-- tests/RuleEligibility.Tests.ps1 | 10 +++++++++- 4 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6f1e0056 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# These inputs are pinned by exact byte hashes; Windows checkouts must keep LF. +/config/security_rules.json text eol=lf +/config/eid_subcategory_mapping.csv text eol=lf +/config/rule_eligibility_manifest.json text eol=lf diff --git a/docs/native-rule-eligibility.md b/docs/native-rule-eligibility.md index 4cd935a5..5d3c21f2 100644 --- a/docs/native-rule-eligibility.md +++ b/docs/native-rule-eligibility.md @@ -22,7 +22,7 @@ The manifest records the exact corpus and EventID-mapping SHA256 values and coun | NotApplicable | All unambiguous, canonical Security event sources belong to other roles than the explicitly selected role. | | Excluded | An explicit Sysmon or identified external-product source. Its ID and exclusion reason remain in the output. | -Every unique rule has reasons and a metadata hash. `PolicyPrerequisites` inventories the catalog's requirements; it is not a separate failed-check verdict. Identical duplicate IDs count once; conflicting duplicate IDs are rejected. The report provides input record count, unique rule count, native candidate count, role-applicable count and exclusion groups. It reports **Ready / native candidates**, **Ready / applicable candidates**, and **Ready / full unique corpus** separately. Empty denominators produce null percentages. Unknown and incomplete native candidates remain in the denominator, including generic categories lacking a verified adapter. These denominators therefore differ from the earlier standalone comparison report's explicitly narrower modeling boundary; do not compare their percentages without reconciling scope and corpus versions. +Every unique rule has reasons and a metadata hash. `ordered-json-html-escaped-utf8-sha256-v1` hashes compact JSON with the fixed field order `id,title,level,category,service,channel,event_ids,subcategory_guids,description,tags`, explicit HTML escaping and UTF-8 without a BOM; this keeps per-rule identities stable across PowerShell 5.1 and 7. Corpus, mapping and imported-artifact hashes always cover the exact file bytes. Git attributes preserve LF checkouts for the shipped pinned inputs, without normalizing imported evidence. `PolicyPrerequisites` inventories the catalog's requirements; it is not a separate failed-check verdict. Identical duplicate IDs count once; conflicting duplicate IDs are rejected. The report provides input record count, unique rule count, native candidate count, role-applicable count and exclusion groups. It reports **Ready / native candidates**, **Ready / applicable candidates**, and **Ready / full unique corpus** separately. Empty denominators produce null percentages. Unknown and incomplete native candidates remain in the denominator, including generic categories lacking a verified adapter. These denominators therefore differ from the earlier standalone comparison report's explicitly narrower modeling boundary; do not compare their percentages without reconciling scope and corpus versions. Category-only GUIDs, blank EventID rows and ambiguous mappings cannot establish subcategory/outcome readiness. In particular, the mapping lists both Token Right Adjusted Events and Authorization Policy Change for 4703; the importer does not arbitrarily choose one. Directory-service and certificate-template change events must be assessed on their source DC, not credited automatically to a member-server CA. diff --git a/modules/RuleEligibility.psm1 b/modules/RuleEligibility.psm1 index 5efc018d..fb9f213e 100644 --- a/modules/RuleEligibility.psm1 +++ b/modules/RuleEligibility.psm1 @@ -57,7 +57,11 @@ function Get-WelaEligibilityRuleHash { foreach ($name in @('id', 'title', 'level', 'category', 'service', 'channel', 'event_ids', 'subcategory_guids', 'description', 'tags')) { $ordered[$name] = $Rule.$name } - Get-WelaEligibilityTextHash (ConvertTo-Json -InputObject $ordered -Depth 12 -Compress) + # Windows PowerShell 5.1 uses HTML escaping by default. Require the same + # spelling on newer editions so identical metadata has one stable digest. + $arguments = @{InputObject=$ordered;Depth=12;Compress=$true} + if ((Get-Command ConvertTo-Json).Parameters.ContainsKey('EscapeHandling')) { $arguments.EscapeHandling = 'EscapeHtml' } + Get-WelaEligibilityTextHash (ConvertTo-Json @arguments) } function Get-WelaEligibilityArtifact { @@ -362,7 +366,7 @@ function Get-WelaRuleEligibility { [pscustomobject][ordered]@{ SchemaVersion = 1; GeneratedAtUtc = $Now.ToString('o'); Scope = 'native-windows-rule-eligibility' AssessmentBasis = $(if ($EvidencePath) { 'Imported lab artifacts; Ready applies only to the recorded context/time and is not a current-host or universal guarantee.' } else { 'Metadata/configuration assessment only; no event-generation, ingestion or query evidence imported.' }) - Corpus = [pscustomobject]@{ Sha256 = $corpusHash; MappingSha256 = $mappingHash; Pinned = [bool]$pinned; Manifest = $manifest; Kind = 'WELA extracted Hayabusa rule metadata; not the complete upstream Sigma corpus' } + Corpus = [pscustomobject]@{ Sha256 = $corpusHash; MappingSha256 = $mappingHash; Pinned = [bool]$pinned; Manifest = $manifest; MetadataHashAlgorithm = 'ordered-json-html-escaped-utf8-sha256-v1'; Kind = 'WELA extracted Hayabusa rule metadata; not the complete upstream Sigma corpus' } RequestedContext = [pscustomobject]@{ Role = $Role; Build = $(if ($Build) { $Build } else { $null }) } Summary = [pscustomobject]@{ InputRecords = $raw.Count; UniqueRules = $rows.Count; DuplicateRecords = $duplicateCount diff --git a/tests/RuleEligibility.Tests.ps1 b/tests/RuleEligibility.Tests.ps1 index 1c177b2c..a64dc570 100644 --- a/tests/RuleEligibility.Tests.ps1 +++ b/tests/RuleEligibility.Tests.ps1 @@ -53,6 +53,13 @@ function Reset-Evidence { } function Assert-NotReady($Message) { Save-Bundle; $r=Report -Evidence; Assert ($r.Summary.Ready -eq 0 -and $r.Results[0].Reasons.Count -gt 0) $Message } try { + # Construct Unicode explicitly so the test source also loads correctly in + # Windows PowerShell 5.1 without relying on a UTF-8 BOM. + $hashVector=Fixture-Rule 'hash-vector' + $hashVector.title='Apostrophe' + [char]0x27 + 's & "quoted" \ path ' + [char]0x65e5 + [char]0x672c + [char]0x8a9e + [char]::ConvertFromUtf32(0x1f600) + $hashVector.description="line`nnext`ttab" + $actualMetadataHash=& (Get-Module RuleEligibility) { param($rule) Get-WelaEligibilityRuleHash $rule } $hashVector + Assert ($actualMetadataHash -eq '8d81c215ae99b303fd30e346613a35b83a1949d7cfa4f436952ef3079b8d0ded') ("Metadata hash is independent of PowerShell JSON escaping: " + $actualMetadataHash) Save-Corpus @(Fixture-Rule) $r=Report Assert ($r.Summary.Ready -eq 0 -and $r.Results[0].State -eq 'Conditional') 'Lossy metadata cannot demonstrate usable rules.' @@ -158,7 +165,8 @@ try { & $exercise -Cmd rule-eligibility -RuleEvidencePath 'operator.json' $timer=[Diagnostics.Stopwatch]::StartNew();$full=Get-WelaRuleEligibility - Assert ($full.Corpus.Pinned -and $full.Summary.UniqueRules -eq $full.Corpus.Manifest.uniqueRuleCount) 'Full shipped corpus and mapping match the manifest.' + $pinDiagnostic='Full shipped corpus/manifest mismatch. Corpus SHA256 actual={0}, expected={1}; mapping SHA256 actual={2}, expected={3}; input count actual={4}, expected={5}; unique count actual={6}, expected={7}.' -f $full.Corpus.Sha256,$full.Corpus.Manifest.corpusSha256,$full.Corpus.MappingSha256,$full.Corpus.Manifest.mappingSha256,$full.Summary.InputRecords,$full.Corpus.Manifest.recordCount,$full.Summary.UniqueRules,$full.Corpus.Manifest.uniqueRuleCount + Assert ($full.Corpus.Pinned -and $full.Summary.InputRecords -eq $full.Corpus.Manifest.recordCount -and $full.Summary.UniqueRules -eq $full.Corpus.Manifest.uniqueRuleCount) $pinDiagnostic Assert ($full.Summary.Ready -eq 0 -and ($full.Summary.NativeCandidates+$full.Summary.Excluded) -eq $full.Summary.UniqueRules) 'Full corpus is partitioned without unverified detection credit.' $json=Join-Path $root 'full.json';$html=Join-Path $root 'full.html' Export-WelaRuleEligibility -Report $full -ResultsPath $json -HtmlPath $html