From f21a9f30e4a327666ecaa09e0b538d7418a0f501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B0=E4=B8=AD=E3=82=B6=E3=83=83=E3=82=AF=20Isaac=20Ma?= =?UTF-8?q?this?= <43838376+Shirofune-Security@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:15:04 +0900 Subject: [PATCH] Add transparent configuration and native rule readiness scores (#417) * Add transparent native audit compliance and evidence readiness scores * Link transparent audit scoring changelog to PR 417 * Resolve scoring outputs against the PowerShell filesystem location --- .gitattributes | 1 + .github/workflows/audit-scoring.yml | 30 +++++ CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + WELA.ps1 | 19 +++ config/audit_scoring.json | 11 ++ docs/audit-scoring.md | 67 ++++++++++ scripts/AuditScoring.ps1 | 174 +++++++++++++++++++++++++ tests/AuditScoring.Tests.ps1 | 164 +++++++++++++++++++++++ tests/AuditScoring.Windows.Tests.ps1 | 60 +++++++++ website/docs/resources/changelog.ja.md | 1 + website/docs/resources/changelog.md | 1 + 12 files changed, 530 insertions(+) create mode 100644 .github/workflows/audit-scoring.yml create mode 100644 config/audit_scoring.json create mode 100644 docs/audit-scoring.md create mode 100644 scripts/AuditScoring.ps1 create mode 100644 tests/AuditScoring.Tests.ps1 create mode 100644 tests/AuditScoring.Windows.Tests.ps1 diff --git a/.gitattributes b/.gitattributes index b8b89ff2..af995659 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,7 @@ /config/security_rules.json text eol=lf /config/eid_subcategory_mapping.csv text eol=lf /config/rule_eligibility_manifest.json text eol=lf +/config/audit_scoring.json text eol=lf # Exact-context default evidence pins these source/collector bytes. /config/baselines.json text eol=lf diff --git a/.github/workflows/audit-scoring.yml b/.github/workflows/audit-scoring.yml new file mode 100644 index 00000000..f8d72eec --- /dev/null +++ b/.github/workflows/audit-scoring.yml @@ -0,0 +1,30 @@ +name: Transparent native audit scoring +on: + push: + branches: ['**'] + pull_request: + workflow_dispatch: +permissions: + contents: read +jobs: + audit-scoring: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [windows-2022, windows-2025] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Arithmetic, provenance and public guards on Windows PowerShell 5.1 + shell: powershell + run: ./tests/AuditScoring.Tests.ps1 + - name: Read-only native score and full corpus on Windows PowerShell 5.1 + shell: powershell + run: ./tests/AuditScoring.Windows.Tests.ps1 + - name: Arithmetic, provenance and public guards on PowerShell 7 + shell: pwsh + run: ./tests/AuditScoring.Tests.ps1 + - name: Read-only native score and full corpus on PowerShell 7 + shell: pwsh + run: ./tests/AuditScoring.Windows.Tests.ps1 diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 222352e7..0ca6d80d 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -4,6 +4,7 @@ **改善:** +- 読み取り専用の `score` JSON・自己完結型 HTML レポートを追加し、高度な監査プロファイルへの適合率と重大度別のネイティブルール検証率を分けて表示します。バージョン付き重み、分子・分母、未確認・除外項目、ソースのハッシュ、記録時の証拠コンテキストを保持します。オフラインでは現在の設定を未確認とし、設定の有効化だけで Ready を加算しません。総合的なセキュリティ評価や Sysmon の検知範囲は示しません。 (#417) (@Shirofune-Security) - 共有する詳細監査プロファイルと優先設定のセキュリティテンプレートについて、オフラインで計画・出力・検証する`gpo-package`を追加しました。省略項目を保持し、成功または失敗だけの最低要件を両方へ拡張する場合は明示指定を求め、未検証のゼロ値による配備は拒否します。出典・申告対象・全項目レビュー・検証済みハッシュを含む配備用ファイルであり、GPOバックアップではありません。正規のGPMC/LGPO準備と未リンクGPO作成のレビュー手順を文書化しました。ドメイン配備と実イベントの検証は別途ラボで必要となり、Sigma検知範囲には加算しません。 (#415) (@Shirofune-Security) - 検証対象のWindows 11クライアント向けに、共通の標準監査プロファイルからオフラインで出力する`intune-export`を追加しました。Microsoft DDFに基づく59件の明示的な対応表、整数型のOMA-URI CSV／Graphデータ、監査サブカテゴリの優先設定、出典と省略理由の一覧を保存します。最小監査マスクは既定で拒否し、`PromoteToBoth`の明示指定時だけ成功・失敗の両方へ拡張します。新規ローカル出力には検証済みハッシュを付け、アップロード・割り当て・ポリシー削除・Windows設定変更は行いません。Intune配備・競合・復旧・イベントの確認は別途必要です。 (#414) (@Shirofune-Security) - `-ProfileFile`で管理者のJSON詳細監査プロファイルを一覧・計画・監査・設定に使用できるようにしました。標準GUID、役割、設定モード、出典ハッシュを厳密に検証し、組み込みプロファイルを保持します。厳密なJSON字句検証で重複キー検出の回避を防ぎ、レポートは新規ファイルに限ることで別名リンク経由でも入力と既存の証拠を保持します。共通の優先設定、復旧記録、変更直前のファイル確認と最終検証を使用し、イベント生成やSigma検知可能性は別途検証とします。 (#416) (@Shirofune-Security) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a765a3c..7e3de412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **Improvements:** +- Added read-only `score` JSON and self-contained HTML reports with separate advanced audit-profile compliance and severity-weighted native rule readiness. Versioned weights, explicit numerators/denominators, unknowns, exclusions, source fingerprints and recorded evidence contexts make each result reviewable. Offline scenarios keep current settings Unknown; enabled settings grant no Ready credit, and no overall security grade or Sysmon coverage is implied. (#417) (@Shirofune-Security) - Added offline `gpo-package` plan, export and verification for shared advanced audit profiles and the precedence security template. Packages preserve omissions, require explicit expansion of one-sided minimum masks, reject unvalidated zero-mask deployment, and include source/target context, full reviews and verified file hashes. These are deployment components, not GPO backups; genuine GPMC/LGPO preparation and reviewed create-unlinked procedures are documented. Native domain application and event evidence remain separate lab acceptance with no Sigma credit. (#415) (@Shirofune-Security) - Added offline `intune-export` for shared native audit profiles on reviewed Windows 11 client targets, with 59 explicit Microsoft DDF mappings, typed OMA-URI CSV/Graph artifacts, the audit precedence prerequisite and complete source/omission manifests. Static minimum masks are rejected unless explicitly expanded with `PromoteToBoth`; fresh local bundles include verified fingerprints and never upload, assign, delete policies or change Windows. Intune deployment, conflicts, recovery and event evidence remain separate validation. (#414) (@Shirofune-Security) - Added `-ProfileFile` for strictly validated custom advanced audit profiles in listing, planning, auditing and configuration. Canonical GUIDs, roles, modes and source hashes remain explicit; built-in profiles are preserved. Strict JSON tokens prevent duplicate-key bypasses, and new report files preserve inputs and prior evidence even through file aliases. Shared precedence, recovery journals, pre-write file checks and final verification protect configuration, without claiming event or Sigma readiness. (#416) (@Shirofune-Security) diff --git a/WELA.ps1 b/WELA.ps1 index ba5905c3..b9f08c71 100644 --- a/WELA.ps1 +++ b/WELA.ps1 @@ -63,6 +63,8 @@ [ValidateSet('OneSettings','SecurityWarning')][string[]]$NotificationControl, [ValidateRange(1,90)][int]$WarningPercent = 90, [switch]$EnablePrivacyChannel, + [string]$ScoreProfile, + [string]$ScoreEvidencePath, [ValidateSet('Plan','Export','Verify')][string]$GpoAction = 'Plan', [string]$GpoProfile, [string]$GpoOutputPath, @@ -112,6 +114,7 @@ Import-Module (Join-Path $ScriptRoot "modules/NativeChannelAccess.psm1") -ErrorA Import-Module (Join-Path $ScriptRoot "modules/WefSubscriptions.psm1") -ErrorAction Stop . (Join-Path $ScriptRoot "scripts/WefDeployment.ps1") . (Join-Path $ScriptRoot "scripts/RetentionHealth.ps1") +. (Join-Path $ScriptRoot "scripts/AuditScoring.ps1") . (Join-Path $ScriptRoot "scripts/TargetedSaclPlanning.ps1") . (Join-Path $ScriptRoot "scripts/GpoAuditPackages.ps1") . (Join-Path $ScriptRoot "scripts/IntuneAuditExport.ps1") @@ -1873,6 +1876,7 @@ Usage: ./WELA.ps1 control-applicability # Read-only historical native feature/build assessment ./WELA.ps1 default-evidence -Help # Exact-context observed snapshots and reviewed reference comparison ./WELA.ps1 audit-notifications -Help # OneSettings audit and Security warning policy + ./WELA.ps1 score -Help # Separate configuration compliance and evidence-qualified readiness ./WELA.ps1 intune-export -Help # Offline native audit OMA-URI/Graph artifacts; no tenant changes ./WELA.ps1 native-validation -Help # Collect a fixed native 4688 probe without changing policy ./WELA.ps1 version # Show the WELA version @@ -1886,6 +1890,13 @@ Write-Host "" Write-Host "WELA v$WELAVersion - $WELAReleaseName" Write-Host "" +if ($Cmd -ne 'score' -and @($PSBoundParameters.Keys | Where-Object { $_ -in @('ScoreProfile','ScoreEvidencePath') }).Count) { + throw 'Scoring options require score. No command was run.' +} +if ($Cmd -eq 'score' -and @($PSBoundParameters.Keys | Where-Object { $_ -notin @('Cmd','ScoreProfile','ScoreEvidencePath','Role','Build','IncludeOptional','ResultsPath','HtmlPath','Help') }).Count) { + throw 'score accepts only score, scenario, optional-selection and report options. No command was run.' +} + if ($Cmd -ne 'gpo-package' -and @($PSBoundParameters.Keys | Where-Object { $_ -in @('GpoAction','GpoProfile','GpoOutputPath','GpoMinimumMode') }).Count) { throw 'GPO package options require gpo-package. No command was run.' } @@ -2004,6 +2015,14 @@ if ($Profile -and $Cmd.ToLower() -in @('plan', 'audit', 'audit-settings', 'confi } switch ($Cmd.ToLower()) { + 'score' { + if ($Help) { Write-Host 'Usage: ./WELA.ps1 score -ScoreProfile profile-id [-Role role -Build build] [-IncludeOptional] [-ScoreEvidencePath evidence.json] [-ResultsPath new.json] [-HtmlPath new.html]. Explicit role/build is an offline scenario; omit both to observe this Windows host. Two separate measures, no overall security grade. See docs/audit-scoring.md.'; return } + if (-not $ScoreProfile) { throw 'score requires an explicit -ScoreProfile. Use profiles to list built-in profiles.' } + $report=Invoke-WelaAuditScore -Profile $ScoreProfile -EvidencePath $ScoreEvidencePath -Role $Role -Build $Build -IncludeOptional:$IncludeOptional + Export-WelaAuditScore -Report $report -ResultsPath $ResultsPath -HtmlPath $HtmlPath + $report.Configuration | Select-Object Label,Numerator,Denominator,Percent,Unknown | Format-List + $report.Readiness | Select-Object Label,Numerator,Denominator,Percent,Ready,ApplicableUniqueRules | Format-List + } 'gpo-package' { if ($Help) { Write-Host 'Usage: ./WELA.ps1 gpo-package [-GpoAction Plan|Export|Verify] [-GpoProfile profile-id -Role Client|MemberServer|DomainController|ADCS -Build number] [-GpoMinimumMode Reject|PromoteToBoth] [-IncludeOptional] [-GpoOutputPath directory] [-DryRun]. Export requires a fresh directory. These are offline components, not an importable GPO backup. See docs/gpo-audit-packages.md.'; return } if ($Profile -or $Baseline -or $HtmlPath -or $Auto -or $BackupPath -or $PlanPath -or $ResultsPath) { throw 'gpo-package uses GpoProfile and GpoOutputPath. Export contains its JSON manifest/review; other profile, result, backup and configuration options are unsupported.' } diff --git a/config/audit_scoring.json b/config/audit_scoring.json new file mode 100644 index 00000000..27681cd5 --- /dev/null +++ b/config/audit_scoring.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, + "id": "native-audit-score-v1", + "version": "1.0.0", + "configurationWeight": 1, + "ruleWeights": {"critical": 20, "high": 15, "medium": 10, "low": 5, "informational": 1, "unknown": 1}, + "configurationScope": "Selected advanced audit subcategories and typed audit precedence only", + "ruleScope": "Applicable unique native candidates; only evidence-qualified Ready earns weight", + "exclusions": ["NotApplicable", "Excluded"], + "note": "Two independent percentages, not an overall security grade. Unknowns remain in each denominator; Sysmon is excluded." +} diff --git a/docs/audit-scoring.md b/docs/audit-scoring.md new file mode 100644 index 00000000..6c29c7a0 --- /dev/null +++ b/docs/audit-scoring.md @@ -0,0 +1,67 @@ +# Transparent native audit scores + +The read-only `score` command reports two independent measures: selected advanced audit-profile configuration compliance and severity-weighted, evidence-qualified native rule readiness. It does not assign an overall security grade. Sysmon is excluded from the readiness measure through the shared native eligibility classifier. + +```powershell +# Observe the actual local Windows host against an explicit built-in profile. +./WELA.ps1 score -ScoreProfile wela-2.2.0 ` + -ResultsPath .\score.json -HtmlPath .\score.html + +# Offline declared scenario, on any platform. No local policy is observed. +./WELA.ps1 score -ScoreProfile wela-2.2.0 -Role Client -Build 26100 ` + -ResultsPath .\client-scenario.json -HtmlPath .\client-scenario.html + +# Assess independently collected complete-rule evidence in its recorded context. +./WELA.ps1 score -ScoreProfile wela-2.2.0 -Role MemberServer -Build 20348 ` + -ScoreEvidencePath .\reviewed-lab\evidence.json -HtmlPath .\lab-readiness.html +``` + +Use `profiles` to list supported built-in IDs and applicability. Specify both Role and Build for a declared offline scenario, or omit both to detect the real Windows host. Supplying Role/Build always selects offline behavior, including on Windows. Offline configuration stays Unknown without credit. Actual observations require a complete, consistent native host context; an unavailable effective-policy reader is reported separately from a known mismatch. Read timestamps and host metadata in the report; a later GPO/MDM refresh can change the settings. No policy is applied, refreshed, cleared or restored by scoring. + +`-ScoreProfile` is required. `-IncludeOptional` selects optional profile controls. Reference-only documentary Windows defaults are not deployable recommendations and are rejected as compliance targets. Custom profile files and custom rule corpora are outside this initial score command. Its dedicated options cannot be used with configuration commands; output files must be new, distinct filesystem paths under existing directories. Relative output paths use PowerShell's current location, including after `Push-Location`; Windows case-variant collisions and non-filesystem providers are rejected before either output is written. + +## Configuration compliance + +Each included advanced audit-policy subcategory has weight 1. Audit precedence (`SCENoApplyLegacyAuditPolicy`, DWORD 1) adds one control when there is a substantive selected audit requirement. The numerator is the number of compliant controls; the denominator includes compliant, drifted and unknown selected controls. + +| Profile entry | Treatment | +| --- | --- | +| Exact mask 0, 1, 2, 3 | Current effective mask must equal the requirement | +| Minimum mask 1, 2, 3 | Current mask must contain all required bits; extra bits are accepted | +| Minimum mask 0 | No constraint; excluded | +| Optional | Excluded unless selected; selected mask is exact | +| Unchanged / Not Configured / Not applicable | Excluded, with the reason retained | +| Unknown current mask or mistyped precedence | Included with zero credit and an Unknown state | + +For example, two compliant audit controls plus unknown precedence produce 2/3, or 66.67%. An all-preserved/minimum-zero profile has no substantive constraints, so precedence is also excluded and the percentage is N/A. A missing precedence value is a known mismatch; an unreadable or incorrectly typed value remains Unknown. + +This percentage covers advanced audit masks and precedence only. It does not score channel enablement, event payload capture, object/AD/WMI SACLs, provider or service activation, AD CS filters, log sizes, retention, access permissions, forwarding, event generation or backend detection. Per-control source IDs and unscored prerequisites remain visible. A 100% configuration result cannot establish that a Sigma rule works. + +## Evidence-qualified rule readiness + +The command freshly invokes `Get-WelaRuleEligibility` against the shipped pinned corpus and mapping. It does not trust an uploaded score summary or use enabled settings as a Ready result. Identical duplicate rule IDs are counted once by the shared classifier; conflicting metadata is rejected. Severity metadata is bound to the exact corpus bytes assessed, and changes to the corpus, mapping or pin manifest during scoring cause failure. + +| Rule severity | Weight | +| --- | ---: | +| Critical | 20 | +| High | 15 | +| Medium | 10 | +| Low | 5 | +| Informational | 1 | +| Missing or unknown | 1, visibly identified | + +The denominator is the sum of weights for applicable unique native candidates. Conditional, Blocked and unknown candidates remain included. Only evidence-qualified Ready rules earn their weight in the numerator. Explicit NotApplicable and Excluded rules—including Sysmon sources—are removed from the denominator and retained in the full list. Zero applicable candidates produces N/A, not 100%. + +For example, one Ready critical rule, one Conditional high rule and one Blocked low rule produce 20/(20+15+5) = 50%. This is a severity-weighted readiness percentage, distinct from the shared classifier's unweighted rule counts and percentages, which remain in JSON. The weights are transparent review priorities, not probabilities, attack coverage or an independent estimate of risk. + +Without a complete supported evidence bundle, readiness earns zero credit. This indicates that usability has not been demonstrated by the supported evidence contract; it does not prove the enabled logs have no detection value. See [native rule eligibility](native-rule-eligibility.md) for the required original rule, complete normalization review, native configuration/event, ingestion and query artifacts. Unsupported rule logic and incomplete evidence remain conditional. Scoring never creates or executes a detection scenario, contacts a backend, translates a query or manufactures results. + +Ready results apply to each artifact's original host, role, patch, backend/version and time. Those contexts can differ across rules and from the machine running the report. The per-rule evidence time, context, reasons and references are preserved; aggregating them does not establish present readiness on a single deployment. + +## Reports, versioning and tests + +JSON retains the definition/version/hash, profile plan and source provenance, actual or declared observation basis, both measures and every underlying eligibility result. The self-contained HTML has two score cards, visible numerators/denominators, unknown/exclusion counts, a configuration table and a rule table with evidence context/time. The observed computer, observation time, detailed host context and policy-read diagnostic are visible separately from historical rule evidence. Arbitrary text is HTML-encoded; no external scripts or assets are loaded. Neither report overwrites existing files. If one output fails after the other has been written, preserve the completed artifact and choose new destinations when retrying. + +Version 1.0.0 is defined in `config/audit_scoring.json` as `native-audit-score-v1`. Changing weights requires a reviewed definition-version change, rather than silently moving the denominator. Exact profile and corpus fingerprints let a reviewer identify what was assessed. The profile hash must match before and after planning and the plan's returned hash; observed source changes abort reporting. Hashes bind recorded content, not the trustworthiness of a malicious evidence author. Letter grades and a combined security score are deliberately not defined by this first implementation of issue #10. + +Tests cover exact/minimum mask truth tables, optional/role omissions, unknown and empty denominators, severity weights, exclusions/unique IDs, evidence-context preservation, source changes, output collisions, HTML encoding and public command isolation. Windows Server 2022/2025 PowerShell 5.1/7 tests read real policy and verify that native masks/precedence remain unchanged. Synthetic Ready rows test arithmetic only. No Windows 11/DC/AD CS deployment or backend query evidence is claimed by those tests. diff --git a/scripts/AuditScoring.ps1 b/scripts/AuditScoring.ps1 new file mode 100644 index 00000000..085f7d2a --- /dev/null +++ b/scripts/AuditScoring.ps1 @@ -0,0 +1,174 @@ +# Read-only, versioned and separable configuration / evidence-qualified readiness measures. +function Get-WelaScoreDefinition { + $path=Join-Path $PSScriptRoot '../config/audit_scoring.json' + $sourceData=Read-WelaScoreJsonBytes $path + $definition=$sourceData.Value + if ($definition.schemaVersion -ne 1 -or $definition.id -cne 'native-audit-score-v1' -or $definition.version -cne '1.0.0' -or $definition.configurationWeight -ne 1) { throw 'Unsupported score definition.' } + $expected=@{critical=20;high=15;medium=10;low=5;informational=1;unknown=1} + if (@($definition.ruleWeights.PSObject.Properties).Count -ne $expected.Count) { throw 'Unexpected severity weights.' } + foreach ($key in $expected.Keys) { if (($definition.ruleWeights.$key -isnot [int] -and $definition.ruleWeights.$key -isnot [long]) -or $definition.ruleWeights.$key -ne $expected[$key]) { throw 'Score weights changed without a reviewed definition version.' } } + $definition | Add-Member NoteProperty Sha256 $sourceData.Sha256 + return $definition +} +function Read-WelaScoreJsonBytes { + param([string]$Path) + $bytes=[IO.File]::ReadAllBytes($Path) + $algorithm=[Security.Cryptography.SHA256]::Create() + try { $hash=([BitConverter]::ToString($algorithm.ComputeHash($bytes))).Replace('-','').ToLowerInvariant() } finally { $algorithm.Dispose() } + $encoding=New-Object Text.UTF8Encoding($false,$true) + # Bind the parsed metadata to these exact bytes, including any BOM/newlines. + $parsed=ConvertFrom-Json -InputObject ($encoding.GetString($bytes).TrimStart([char]0xFEFF)) -ErrorAction Stop + [pscustomobject]@{Sha256=$hash;Value=$parsed} +} +function Get-WelaScorePercent { + param([long]$Numerator,[long]$Denominator) + if ($Denominator -eq 0) { return $null } + return [Math]::Round(100.0*$Numerator/$Denominator,2) +} +function Get-WelaConfigurationScore { + param($Plan,$Precedence) + if ($Plan.referenceOnly) { throw 'Reference-only documentary defaults are not a recommended compliance target.' } + $rows=@(foreach ($policy in $Plan.policies) { + $included=$policy.mode -in @('exact','minimum') -or ($policy.mode -eq 'optional' -and $Plan.includeOptional) + $reason='Selected advanced audit-policy requirement.' + if ($included -and (($policy.requiredMask -isnot [int] -and $policy.requiredMask -isnot [long]) -or $policy.requiredMask -notin @(0,1,2,3))) { throw 'Invalid selected requirement mask.' } + if (-not $included) { $reason="Omitted from denominator: $($policy.mode)." } + elseif ($policy.mode -eq 'minimum' -and $policy.requiredMask -eq 0) { $included=$false; $reason='Minimum zero imposes no constraint; omitted from denominator.' } + $state='Excluded'; $earned=0 + if ($included) { + $current=$policy.currentMask; $required=$policy.requiredMask + if (($required -isnot [int] -and $required -isnot [long]) -or $required -notin @(0,1,2,3)) { throw 'Invalid selected requirement mask.' } + if (($current -isnot [int] -and $current -isnot [long]) -or $current -notin @(0,1,2,3)) { $state='Unknown'; $reason='Effective policy is unknown; included without credit.' } + else { + $matches=if ($policy.mode -eq 'minimum') { ($current -band $required) -eq $required } else { $current -eq $required } + if ($matches) { $state='Compliant'; $earned=1 } else { $state='Drift' } + } + } + [pscustomobject]@{Id=$policy.id;Guid=$policy.guid;Mode=$policy.mode;Current=$policy.currentMask;Required=$policy.requiredMask;Included=$included;State=$state;Weight=([int]$included);Earned=$earned;Reason=$reason;Prerequisites=$policy.prerequisites;Sources=$policy.sourceIds} + }) + $requiresPrecedence=@($rows | Where-Object Included).Count -gt 0 + $state='Unknown'; $reason='Typed audit precedence is unknown; included without credit.'; $earned=0 + $registry=$Precedence.Registry + if ($registry) { + if (-not $registry.ValueExists) { $state='Drift'; $reason='Audit precedence is not configured.' } + elseif ($registry.Type -eq 'DWord' -and ($registry.Value -is [int] -or $registry.Value -is [long]) -and $registry.Value -in @(0,1)) { + if ($registry.Value -eq 1) { $state='Compliant'; $earned=1; $reason='Observed typed audit precedence matches.' } else { $state='Drift'; $reason='Audit precedence is disabled.' } + } + } + if (-not $requiresPrecedence) { $state='Excluded'; $earned=0; $reason='No selected substantive audit constraint requires precedence; omitted from denominator.' } + $rows+=[pscustomobject]@{Id='SCENoApplyLegacyAuditPolicy';Guid=$null;Mode='exact';Current=$registry;Required='DWORD=1';Included=$requiresPrecedence;State=$state;Weight=([int]$requiresPrecedence);Earned=$earned;Reason=$reason;Prerequisites='Later GPO/MDM refresh can change policy.';Sources=@('Advanced audit policy precedence prerequisite')} + $included=@($rows | Where-Object Included) + $numerator=[long](@($rows | Measure-Object Earned -Sum)[0].Sum) + $denominator=[long]$included.Count + [pscustomobject]@{Label='Advanced audit profile configuration compliance';Numerator=$numerator;Denominator=$denominator;Percent=(Get-WelaScorePercent $numerator $denominator);Compliant=@($included | Where-Object State -eq 'Compliant').Count;Drift=@($included | Where-Object State -eq 'Drift').Count;Unknown=@($included | Where-Object State -eq 'Unknown').Count;Excluded=@($rows | Where-Object { -not $_.Included }).Count;Scope='Advanced audit subcategories plus precedence. Channels, SACLs, provider activation, retention, forwarding and event generation are not scored.';Rows=$rows} +} +function Get-WelaReadinessScore { + param($Eligibility,[hashtable]$Levels,$Definition) + $seen=@{} + $rows=@(foreach ($rule in $Eligibility.Results) { + if ($rule.Id -isnot [string] -or [string]::IsNullOrWhiteSpace($rule.Id)) { throw 'Eligibility rule ID is missing.' } + if ($seen.ContainsKey([string]$rule.Id)) { throw 'Duplicate rule in eligibility output.' }; $seen[[string]$rule.Id]=$true + $level=[string]$Levels[[string]$rule.Id]; $weightLevel=$level.ToLowerInvariant() + if ($weightLevel -notin @('critical','high','medium','low','informational')) { $weightLevel='unknown' } + $weight=[long]$Definition.ruleWeights.$weightLevel + $included=$rule.State -notin @('Excluded','NotApplicable') + $earned=if ($included -and $rule.State -eq 'Ready') { $weight } else { 0 } + [pscustomobject]@{Id=$rule.Id;Title=$rule.Title;Level=$level;WeightLevel=$weightLevel;Weight=$weight;Included=$included;Earned=$earned;State=$rule.State;Reasons=$rule.Reasons;ScopeExclusion=$rule.ScopeExclusion;EvidenceAsOfUtc=$rule.EvidenceAsOfUtc;EvidenceContext=$rule.EvidenceContext;EvidenceArtifacts=$rule.EvidenceArtifacts;MetadataSha256=$rule.MetadataSha256} + }) + $included=@($rows | Where-Object Included) + $numerator=[long](@($included | Measure-Object Earned -Sum)[0].Sum) + $denominator=[long](@($included | Measure-Object Weight -Sum)[0].Sum) + [pscustomobject]@{Label='Evidence-qualified native rule readiness (severity weighted)';Numerator=$numerator;Denominator=$denominator;Percent=(Get-WelaScorePercent $numerator $denominator);ApplicableUniqueRules=$included.Count;Ready=@($included | Where-Object State -eq 'Ready').Count;UnknownSeverity=@($included | Where-Object WeightLevel -eq 'unknown').Count;Excluded=@($rows | Where-Object { -not $_.Included }).Count;Scope='Ready applies only to the original recorded evidence context and time. No evidence yields zero credit, not proof that logging is ineffective.';Rows=$rows} +} +function Invoke-WelaAuditScore { + param([Parameter(Mandatory)][string]$Profile,[string]$EvidencePath,[string]$Role,[int]$Build,[switch]$IncludeOptional) + if (($Role -and -not $Build) -or ($Build -and -not $Role)) { throw 'Supply both Role and Build for an offline scenario, or neither for the actual Windows host.' } + $definition=Get-WelaScoreDefinition + $current=@{}; $precedence=Get-WelaAuditPrecedenceState -Offline + $observation=[pscustomobject]@{Basis='Offline scenario; no native settings observed';ComputerName=$null;Context=$null;Diagnostic='';CapturedUtc=[DateTime]::UtcNow.ToString('o')} + if (-not $Role) { + $hostContext=Get-WelaHostContext; $Role=$hostContext.Role; $Build=$hostContext.Build + $observation.Context=Get-WelaDefaultContext + $observation.Basis='Actual native Windows observation' + $observation.ComputerName=[Environment]::MachineName + if (-not (Test-WelaDefaultContextComplete $observation.Context) -or $observation.Context.Build -ne $Build) { throw 'Complete consistent actual host context is required; use explicit Role/Build for an offline scenario.' } + $detailed=$observation.Context + $roleMatches=switch ($Role) { + 'Client' { $detailed.ProductType -eq 1 -and $detailed.DomainRole -in @(0,1) } + 'DomainController' { $detailed.ProductType -eq 2 -and $detailed.DomainRole -in @(4,5) } + 'MemberServer' { $detailed.ProductType -eq 3 -and $detailed.DomainRole -in @(2,3) } + 'ADCS' { $detailed.ProductType -eq 3 -and $detailed.DomainRole -in @(2,3) -and $detailed.InstalledRoles -contains 'ADCS-Cert-Authority' } + default { $false } + } + if (-not $roleMatches) { throw 'Observed score role contradicts detailed native host context.' } + try { $current=Get-WelaEffectiveAuditPolicy } catch { $observation.Diagnostic=$_.Exception.Message } + $precedence=Get-WelaAuditPrecedenceState + } + $profilePath=Join-Path $PSScriptRoot '../config/audit_profiles.json' + $profileHash=(Get-FileHash -LiteralPath $profilePath -Algorithm SHA256).Hash.ToLowerInvariant() + $plan=Get-WelaAuditProfilePlan -Profile $Profile -Role $Role -Build $Build -Current $current -IncludeOptional:$IncludeOptional + if ($profileHash -cne $plan.schemaSha256.ToLowerInvariant() -or $profileHash -cne (Get-FileHash -LiteralPath $profilePath -Algorithm SHA256).Hash.ToLowerInvariant()) { throw 'Profile source changed during scoring; its plan provenance is not stable.' } + $configuration=Get-WelaConfigurationScore -Plan $plan -Precedence $precedence + $corpusPath=Join-Path $PSScriptRoot '../config/security_rules.json' + $mappingPath=Join-Path $PSScriptRoot '../config/eid_subcategory_mapping.csv' + $manifestPath=Join-Path $PSScriptRoot '../config/rule_eligibility_manifest.json' + $sourceData=Read-WelaScoreJsonBytes $corpusPath + $beforeHash=$sourceData.Sha256; $metadata=$sourceData.Value + $mappingHash=(Get-FileHash -LiteralPath $mappingPath -Algorithm SHA256).Hash.ToLowerInvariant() + $manifestHash=(Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256).Hash.ToLowerInvariant() + $arguments=@{Role=$Role;Build=$Build} + if ($EvidencePath) { $arguments.EvidencePath=$EvidencePath } + $eligibility=Get-WelaRuleEligibility @arguments + $afterHash=(Get-FileHash -LiteralPath $corpusPath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($beforeHash -cne $afterHash -or $beforeHash -cne $eligibility.Corpus.Sha256 -or + $eligibility.Corpus.Pinned -isnot [bool] -or -not $eligibility.Corpus.Pinned -or + $mappingHash -cne $eligibility.Corpus.MappingSha256 -or + $mappingHash -cne (Get-FileHash -LiteralPath $mappingPath -Algorithm SHA256).Hash.ToLowerInvariant() -or + $manifestHash -cne (Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256).Hash.ToLowerInvariant()) { + throw 'Rule corpus/mapping/manifest changed or is unpinned; severity metadata is not bound to this eligibility report.' + } + $levels=@{} + foreach ($entry in $metadata) { + if ($levels.ContainsKey([string]$entry.id) -and $levels[[string]$entry.id] -cne [string]$entry.level) { throw 'Conflicting duplicate severity metadata.' } + $levels[[string]$entry.id]=[string]$entry.level + } + $readiness=Get-WelaReadinessScore -Eligibility $eligibility -Levels $levels -Definition $definition + [pscustomobject][ordered]@{SchemaVersion=1;Kind='WelaNativeAuditScores';GeneratedUtc=[DateTime]::UtcNow.ToString('o');Definition=$definition;Observation=$observation;ProfilePlan=$plan;Precedence=$precedence;Configuration=$configuration;Readiness=$readiness;Eligibility=$eligibility;OverallGrade=$null;Note=$definition.note} +} +function Export-WelaAuditScore { + param($Report,[string]$ResultsPath,[string]$HtmlPath) + $resolved=@{ResultsPath=$null;HtmlPath=$null} + foreach ($name in @('ResultsPath','HtmlPath')) { + $candidate=if ($name -eq 'ResultsPath') { $ResultsPath } else { $HtmlPath } + if ([string]::IsNullOrWhiteSpace($candidate)) { continue } + $provider=$null; $drive=$null + $full=$ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($candidate,[ref]$provider,[ref]$drive) + if ($provider.Name -ne 'FileSystem') { throw 'Score output requires a FileSystem provider path.' } + $resolved[$name]=[IO.Path]::GetFullPath($full) + } + $ResultsPath=$resolved.ResultsPath; $HtmlPath=$resolved.HtmlPath + $comparison=if ([Environment]::OSVersion.Platform -eq [PlatformID]::Win32NT) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + if ($ResultsPath -and $HtmlPath -and [string]::Equals($ResultsPath,$HtmlPath,$comparison)) { throw 'JSON and HTML destinations must differ.' } + $paths=@($ResultsPath,$HtmlPath | Where-Object { $_ }) + foreach ($path in $paths) { if (Test-Path -LiteralPath $path) { throw 'Score output must use new files.' }; if (-not (Test-Path -LiteralPath (Split-Path $path -Parent) -PathType Container)) { throw 'Score output parent must exist.' } } + $outputs=@() + if ($ResultsPath) { $outputs+=@{Path=$ResultsPath;Text=($Report | ConvertTo-Json -Depth 28)} } + if ($HtmlPath) { + $e={param($value) [Net.WebUtility]::HtmlEncode([string]$value)} + $cards=foreach ($measure in @($Report.Configuration,$Report.Readiness)) { + $percent=if ($null -eq $measure.Percent) { 'N/A' } else { [string]::Format([Globalization.CultureInfo]::InvariantCulture,'{0:0.##}%',$measure.Percent) } + $bar=if ($null -eq $measure.Percent) { 0 } else { $measure.Percent } + '

'+(& $e $measure.Label)+'

'+$percent+'

'+(& $e "$($measure.Numerator) / $($measure.Denominator) weighted points")+'

'+(& $e $measure.Scope)+'

' + } + $configRows=foreach ($row in $Report.Configuration.Rows) { ''+(& $e $row.Id)+''+(& $e $row.Mode)+''+(& $e $row.State)+''+(& $e "$($row.Earned)/$($row.Weight)")+''+(& $e $row.Reason)+''+(& $e $row.Prerequisites)+'' } + $ruleRows=foreach ($row in $Report.Readiness.Rows) { ''+(& $e $row.Id)+''+(& $e $row.Title)+''+(& $e $row.WeightLevel)+''+(& $e $row.State)+''+(& $e "$($row.Earned)/$($row.Weight); included=$($row.Included)")+''+(& $e ($row.Reasons -join '; '))+''+(& $e $row.EvidenceAsOfUtc)+''+(& $e ($row.EvidenceContext | ConvertTo-Json -Depth 8 -Compress))+'' } + $observation='

Observed computer: '+(& $e $Report.Observation.ComputerName)+'

Observation UTC: '+(& $e $Report.Observation.CapturedUtc)+'

Observation diagnostic: '+(& $e $Report.Observation.Diagnostic)+'

Detailed observed context
'+(& $e ($Report.Observation.Context | ConvertTo-Json -Depth 8))+'
' + $html='WELA native audit scores

WELA native audit scores

'+(& $e $Report.Note)+'

'+(& $e "$($Report.Observation.Basis). Profile $($Report.ProfilePlan.profile); $($Report.ProfilePlan.role), build $($Report.ProfilePlan.build).")+'

'+$observation+'
'+($cards -join '')+'

'+(& $e "Configuration: $($Report.Configuration.Unknown) unknown, $($Report.Configuration.Excluded) excluded. Rules: $($Report.Readiness.Ready) Ready of $($Report.Readiness.ApplicableUniqueRules) applicable unique candidates; $($Report.Readiness.Excluded) excluded, $($Report.Readiness.UnknownSeverity) unknown severity. Evidence is tied to its recorded context/time, not automatically this host.")+'

Version '+(& $e $Report.Definition.version)+'; definition SHA-256 '+(& $e $Report.Definition.Sha256)+'; corpus SHA-256 '+(& $e $Report.Eligibility.Corpus.Sha256)+'.

Configuration controls

'+($configRows -join '')+'
ControlModeStatePointsReasonUnscored prerequisite

Native rules and exclusions

Severity weights: critical 20, high 15, medium 10, low 5, informational 1, unknown 1. Only Ready earns points. Excluded rows never enter the denominator.

'+($ruleRows -join '')+'
Rule IDTitleSeverityStatePointsReasonsEvidence timeEvidence context
' + $outputs+=@{Path=$HtmlPath;Text=$html} + } + foreach ($output in $outputs) { + $bytes=[Text.UTF8Encoding]::new($false).GetBytes([string]$output.Text) + $stream=[IO.File]::Open($output.Path,[IO.FileMode]::CreateNew,[IO.FileAccess]::Write,[IO.FileShare]::None) + try { $stream.Write($bytes,0,$bytes.Length) } finally { $stream.Dispose() } + } +} diff --git a/tests/AuditScoring.Tests.ps1 b/tests/AuditScoring.Tests.ps1 new file mode 100644 index 00000000..31a1cb37 --- /dev/null +++ b/tests/AuditScoring.Tests.ps1 @@ -0,0 +1,164 @@ +$ErrorActionPreference='Stop' +$root=Split-Path $PSScriptRoot -Parent +. (Join-Path $root 'scripts/AuditScoring.ps1') +. (Join-Path $root 'scripts/ControlApplicability.ps1') +Import-Module (Join-Path $root 'modules/AuditProfiles.psm1') -Force +$script:count=0 +function Assert($Condition,$Message) { if (-not $Condition) {throw $Message};$script:count++ } +function Throws($Action,$Pattern) { $message='';try {& $Action | Out-Null}catch{$message=$_.Exception.Message};Assert ($message -match $Pattern) "Expected $Pattern; got $message" } +function Policy($Mode,$Required,$Current) {[pscustomobject]@{id='fixture';guid='fixture';mode=$Mode;requiredMask=$Required;currentMask=$Current;prerequisites='SACL unscored';sourceIds=@('fixture')}} +function Plan($Rows,[switch]$Optional) {[pscustomobject]@{referenceOnly=$false;includeOptional=[bool]$Optional;policies=@($Rows)}} +$precedence=[pscustomobject]@{Registry=[pscustomobject]@{ValueExists=$true;Type='DWord';Value=1}} +$definition=Get-WelaScoreDefinition +Assert ($definition.Sha256 -ceq (Get-FileHash -LiteralPath (Join-Path $root 'config/audit_scoring.json')).Hash.ToLowerInvariant()) 'Definition fingerprint covers the exact bytes parsed.' +foreach ($required in 0..3) {foreach ($current in 0..3) { + $score=Get-WelaConfigurationScore (Plan @(Policy exact $required $current)) $precedence + $matches=@('0/0','1/1','2/2','3/3') -contains "$required/$current" + Assert ($score.Denominator -eq 2 -and $score.Rows[0].Earned -eq [int]$matches -and $score.Numerator -eq (1+[int]$matches)) "Exact $required/$current truth table including precedence." +}} +foreach ($required in 0..3) {foreach ($current in 0..3) { + $score=Get-WelaConfigurationScore (Plan @(Policy minimum $required $current)) $precedence + if ($required -eq 0) {Assert ($score.Denominator -eq 0 -and $null -eq $score.Percent) 'Minimum zero and unnecessary precedence impose no scored constraint.'} + else { + $matches=@('1/1','1/3','2/2','2/3','3/3') -contains "$required/$current" + Assert ($score.Denominator -eq 2 -and $score.Rows[0].Earned -eq [int]$matches) "Minimum $required/$current includes compliant supersets." + } +}} +foreach ($current in @($null,'1',$true,1.5,-1,4)) { + $score=Get-WelaConfigurationScore (Plan @(Policy exact 1 $current)) $precedence + Assert ($score.Unknown -eq 1 -and $score.Denominator -eq 2 -and $score.Numerator -eq 1) 'Unknown/malformed effective values remain in the denominator with zero credit.' +} +foreach ($mode in @('exact','minimum','optional')) { + Throws {Get-WelaConfigurationScore (Plan @(Policy $mode '0' 0) -Optional) $precedence} 'Invalid selected' +} +foreach ($mode in @('unchanged','not-configured','not-applicable','optional')) { + $score=Get-WelaConfigurationScore (Plan @(Policy $mode 3 3)) $precedence + Assert ($score.Denominator -eq 0 -and $score.Numerator -eq 0 -and $null -eq $score.Percent) 'Preserve, N/A and unselected optional rows do not improve scores.' +} +$score=Get-WelaConfigurationScore (Plan @(Policy optional 1 3) -Optional) $precedence +Assert ($score.Denominator -eq 2 -and $score.Rows[0].State -eq 'Drift') 'Selected optional mask is exact, not minimum.' +foreach ($registry in @($null,[pscustomobject]@{ValueExists=$true;Type='String';Value='1'},[pscustomobject]@{ValueExists=$true;Type='DWord';Value='1'})) { + $score=Get-WelaConfigurationScore (Plan @(Policy exact 1 1)) ([pscustomobject]@{Registry=$registry}) + Assert ($score.Unknown -eq 1 -and $score.Denominator -eq 2 -and $score.Percent -eq 50) 'Unknown/mistyped precedence stays in denominator without credit.' +} +$reference=Plan @();$reference.referenceOnly=$true +Throws {Get-WelaConfigurationScore $reference $precedence} 'Reference-only' +Assert ($null -eq (Get-WelaScorePercent 0 0)) 'Empty denominator is null, never perfect or zero percent.' +function Rule($Id,$State='Conditional') {[pscustomobject]@{Id=$Id;Title='Fixture ';State=$State;Reasons=@('Evidence unverified ');ScopeExclusion=$null;EvidenceAsOfUtc=$null;EvidenceContext=$null;EvidenceArtifacts=@();MetadataSha256=('a'*64);ConfigurationEstimate=$true}} +$levels=@{a='critical';b='high';c='medium';d='low';e='informational';f='unknown';g='';h='CRITICAL'} +$rules=@(foreach($id in @('a','b','c','d','e','f','g','h')){Rule $id}) +$readiness=Get-WelaReadinessScore ([pscustomobject]@{Results=$rules}) $levels $definition +Assert ($readiness.Denominator -eq 73 -and $readiness.Numerator -eq 0 -and $readiness.UnknownSeverity -eq 2) 'All severity weights, including unknown=1, are explicit; configuration hints earn no readiness credit.' +$rules[0].State='Ready';$rules[0].EvidenceAsOfUtc='2026-09-19T10:00:00Z';$rules[0].EvidenceContext=[pscustomobject]@{computer='lab<&>';role='Client';build=26100;patch='fixture';backend='fixture-backend';backendVersion='1'} +$rules[0].EvidenceArtifacts=@([pscustomobject]@{path='event.xml';sha256=('b'*64)}) +$rules[1].State='NotApplicable';$rules[1].ScopeExclusion='Different role';$rules[2].State='Excluded';$rules[2].ScopeExclusion='Sysmon';$rules[3].State='Blocked' +$readiness=Get-WelaReadinessScore ([pscustomobject]@{Results=$rules}) $levels $definition +Assert ($readiness.Denominator -eq 48 -and $readiness.Numerator -eq 20 -and $readiness.Ready -eq 1 -and $readiness.Excluded -eq 2) 'Only evidence-qualified Ready earns weight; Blocked/Conditional stay in scope and explicit exclusions are removed.' +Assert ($readiness.Rows[0].EvidenceContext.computer -ceq 'lab<&>' -and $readiness.Rows[0].EvidenceArtifacts[0].path -eq 'event.xml') 'Recorded evidence context and artifact references survive scoring.' +Throws {Get-WelaReadinessScore ([pscustomobject]@{Results=@($rules[0],$rules[0])}) $levels $definition} 'Duplicate' +Throws {Get-WelaReadinessScore ([pscustomobject]@{Results=@(Rule '')}) $levels $definition} 'ID is missing' +$empty=Get-WelaReadinessScore ([pscustomobject]@{Results=@()}) @{} $definition +Assert ($empty.Denominator -eq 0 -and $null -eq $empty.Percent) 'Empty readiness scope is N/A.' +$none=Get-WelaReadinessScore ([pscustomobject]@{Results=@((Rule x Excluded),(Rule y NotApplicable))}) @{} $definition +Assert ($none.Denominator -eq 0 -and $null -eq $none.Percent) 'All-excluded readiness scope is N/A.' +$temp=Join-Path ([IO.Path]::GetTempPath()) ('wela-score-'+[guid]::NewGuid().ToString('N'));$null=New-Item -ItemType Directory -Path $temp +try { + $report=[pscustomobject]@{Note='Two measures ';Definition=$definition;Observation=[pscustomobject]@{Basis='Offline ';CapturedUtc='2026-09-19T09:00:00Z';Diagnostic='Access denied ';Context=[pscustomobject]@{Computer='observed-fixture'}};ProfilePlan=[pscustomobject]@{profile='fixture';role='Client';build=26100};Configuration=$score;Readiness=$readiness;Eligibility=[pscustomobject]@{Corpus=[pscustomobject]@{Sha256=('c'*64)}};OverallGrade=$null} + $json=Join-Path $temp 'score.json';$html=Join-Path $temp 'score.html' + Export-WelaAuditScore $report $json $html + $loaded=Get-Content -LiteralPath $json -Raw | ConvertFrom-Json + $rendered=[IO.File]::ReadAllText($html) + Assert ($loaded.Readiness.Numerator -eq 20 -and $loaded.Readiness.Rows[0].EvidenceContext.computer -ceq 'lab<&>' -and $null -eq $loaded.OverallGrade) 'JSON retains separate measures and historical evidence scope, without an overall grade.' + Assert ($rendered -notmatch '