C64-dev/newLang/tests/regression/run-regression.ps1

# E08-S01: automated x128 regression harness.
#
# Runs each suite from a manifest in its own child PowerShell, with a fresh
# build directory (so every suite creates its own D81), a wall-clock limit that
# also bounds emulated cycles, a captured log, and a machine-readable results
# file. When a suite declares artifacts, their SHA-256 values are compared with
# a reviewed baseline; -UpdateBaseline rewrites that baseline.
[CmdletBinding()]
param(
    [string]$Manifest = "",
    [string]$BuildRoot = "",
    [string]$Baseline = "",
    [Alias("Results")][string]$ResultsPath = "",
    [string[]]$Filter = @(),
    [switch]$UpdateBaseline,
    [switch]$NoFresh
)

$ErrorActionPreference = "Stop"

$project = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent
if ($Manifest -eq "") { $Manifest = Join-Path $PSScriptRoot "suites.ps1" }
if ($BuildRoot -eq "") { $BuildRoot = Join-Path $project "build" }
if ($Baseline -eq "") { $Baseline = Join-Path $PSScriptRoot "baseline.json" }
if ($ResultsPath -eq "") { $ResultsPath = Join-Path $BuildRoot "regression\results.json" }

foreach ($path in @($Manifest)) {
    if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "manifest not found: $path" }
}
$suites = @(& $Manifest)
if ($suites.Count -eq 0) { throw "manifest lists no suites: $Manifest" }
# -File invocation passes "source,lexer" as one string, so accept both forms.
$wanted = @()
foreach ($item in $Filter) { $wanted += @($item -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) }
if ($wanted.Count -gt 0) {
    $suites = @($suites | Where-Object { $wanted -contains $_.Name })
    if ($suites.Count -eq 0) { throw "filter matched no suite: $($wanted -join ', ')" }
}

$logDirectory = Join-Path $BuildRoot "regression"
New-Item -ItemType Directory -Force -Path $logDirectory | Out-Null
$baselineData = @{}
if (Test-Path -LiteralPath $Baseline -PathType Leaf) {
    $raw = Get-Content -Raw -LiteralPath $Baseline | ConvertFrom-Json
    foreach ($property in $raw.PSObject.Properties) {
        $entries = @{}
        foreach ($artifact in $property.Value.PSObject.Properties) {
            $entries[$artifact.Name] = [string]$artifact.Value
        }
        $baselineData[$property.Name] = $entries
    }
}

# The first line that looks like a diagnosis, so a failed suite names the
# fixture and what was observed instead of only an exit code.
function Get-Signature {
    param([AllowEmptyString()][string]$Text)
    $lines = @($Text -split "`r?`n" | Where-Object { $_.Trim() -ne "" })
    if ($lines.Count -eq 0) { return "" }
    $hit = $lines | Where-Object { $_ -match "(?i)fail|differs|error|exception|missing|did not|refus" } |
        Select-Object -First 1
    if (-not $hit) { $hit = $lines | Select-Object -Last 1 }
    $hit = $hit.Trim()
    if ($hit.Length -gt 240) { $hit = $hit.Substring(0, 240) }
    return $hit
}

function Get-ArtifactFiles {
    param([string]$Directory, [string[]]$Patterns)
    if (-not $Directory -or $Patterns.Count -eq 0) { return @() }
    if (-not (Test-Path -LiteralPath $Directory -PathType Container)) { return @() }
    $extensions = @($Patterns | ForEach-Object { $_.ToLowerInvariant() })
    $files = Get-ChildItem -LiteralPath $Directory -File -Recurse |
        Where-Object { $extensions -contains ("*" + $_.Extension.ToLowerInvariant()) }
    return @($files | Sort-Object FullName)
}

$results = @()
$startedAt = (Get-Date).ToString("o")

foreach ($suite in $suites) {
    $name = [string]$suite.Name
    $script = [string]$suite.Script
    if (-not [IO.Path]::IsPathRooted($script)) { $script = Join-Path $project $script }
    $timeout = if ($suite.ContainsKey("Timeout")) { [int]$suite.Timeout } else { 300 }
    $buildDirectory = $null
    if ($suite.ContainsKey("BuildDir") -and $suite.BuildDir) {
        $buildDirectory = Join-Path $BuildRoot ([string]$suite.BuildDir)
    }

    $problems = @()
    if (-not (Test-Path -LiteralPath $script -PathType Leaf)) {
        $problems += ("suite script missing: $script")
    }
    if ($problems.Count -eq 0 -and $buildDirectory -and -not $NoFresh) {
        if (Test-Path -LiteralPath $buildDirectory) {
            Remove-Item -LiteralPath $buildDirectory -Recurse -Force
        }
    }

    $log = Join-Path $logDirectory ("$name.log")
    $errorLog = Join-Path $logDirectory ("$name.err.log")
    foreach ($path in @($log, $errorLog)) {
        if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Force }
    }

    $seconds = 0
    $exitCode = -1
    $timedOut = $false
    $text = ""
    if ($problems.Count -eq 0) {
        $started = Get-Date
        $psi = [Diagnostics.ProcessStartInfo]::new()
        $psi.FileName = (Get-Command powershell).Source
        $psi.Arguments = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", ('"' + $script + '"')) -join ' '
        $psi.UseShellExecute = $false
        $psi.CreateNoWindow = $true
        $psi.RedirectStandardOutput = $true
        $psi.RedirectStandardError = $true
        $process = [Diagnostics.Process]::Start($psi)
        $outTask = $process.StandardOutput.ReadToEndAsync()
        $errTask = $process.StandardError.ReadToEndAsync()
        if (-not $process.WaitForExit($timeout * 1000)) {
            $timedOut = $true
            & taskkill /PID $process.Id /T /F 2>&1 | Out-Null
            if (-not $process.WaitForExit(10000)) { $process.Kill() }
        }
        $seconds = [Math]::Round(((Get-Date) - $started).TotalSeconds, 1)
        $exitCode = if ($timedOut) { -1 } else { $process.ExitCode }
        $text = ($outTask.Result + "`n" + $errTask.Result)
        [IO.File]::WriteAllText($log, $text)
        if ($timedOut) {
            $problems += ("timed out after $timeout s (cycle budget exhausted)")
        } elseif ($exitCode -ne 0) {
            $problems += ("exit code $exitCode")
        }
        if (-not $timedOut -and $suite.ContainsKey("Success") -and $suite.Success -and
            $text -notmatch [string]$suite.Success) {
            $problems += ("missing success marker '$($suite.Success)'")
        }
    }

    # Artifact capture and comparison.
    $artifacts = @()
    $patterns = @()
    if ($suite.ContainsKey("Artifacts")) { $patterns = @($suite.Artifacts) }
    if ($buildDirectory -and $patterns.Count -gt 0) {
        $expected = if ($baselineData.ContainsKey($name)) { $baselineData[$name] } else { @{} }
        foreach ($file in (Get-ArtifactFiles -Directory $buildDirectory -Patterns $patterns)) {
            $relative = $file.FullName.Substring($buildDirectory.Length).TrimStart('\', '/')
            $hash = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256).Hash
            $state = if (-not $expected.ContainsKey($relative)) {
                "new"
            } elseif ($expected[$relative] -ne $hash) {
                "changed"
            } else {
                "match"
            }
            if (-not $UpdateBaseline -and $state -ne "match" -and $baselineData.ContainsKey($name)) {
                $problems += ("artifact ${relative}: $state")
            }
            $artifacts += ([ordered]@{
                path = $relative; sha256 = $hash; status = $state
            })
        }
        foreach ($relative in $expected.Keys) {
            if (-not ($artifacts | Where-Object { $_.path -eq $relative })) {
                if (-not $UpdateBaseline) { $problems += ("artifact ${relative}: missing") }
                $artifacts += ([ordered]@{ path = $relative; sha256 = ""; status = "missing" })
            }
        }
    }

    $status = if ($problems.Count -eq 0) { "pass" } else { "fail" }
    # A bare exit status says nothing about the fixture; prefer the diagnosis
    # the suite printed, and fall back to the structural problem when it
    # produced no output at all.
    $observed = Get-Signature $text
    $signature = if ($problems.Count -eq 0) { $observed }
        elseif ($problems[0] -match "^exit code" -and $observed) { $observed }
        else { $problems[0] }
    $results += ([ordered]@{
        name = $name
        status = $status
        seconds = $seconds
        exitCode = $exitCode
        timedOut = $timedOut
        signature = $signature
        problems = @($problems)
        log = $log
        artifacts = @($artifacts)
    })
    Write-Host ("  {0,-14} {1,-4} {2,6}s  {3}" -f $name, $status, $seconds, $signature)
}

$finishedAt = (Get-Date).ToString("o")
$report = [ordered]@{
    startedAt = $startedAt
    finishedAt = $finishedAt
    manifest = $Manifest
    suites = @($results)
    passed = @($results | Where-Object { $_.status -eq "pass" }).Count
    failed = @($results | Where-Object { $_.status -eq "fail" }).Count
}
$resultsDirectory = Split-Path $ResultsPath -Parent
New-Item -ItemType Directory -Force -Path $resultsDirectory | Out-Null
$report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $ResultsPath

if ($UpdateBaseline) {
    $fresh = [ordered]@{}
    foreach ($entry in $results) {
        if ($entry.status -ne "pass" -or $entry.artifacts.Count -eq 0) { continue }
        $map = [ordered]@{}
        foreach ($artifact in $entry.artifacts) {
            $map[$artifact.path] = $artifact.sha256
        }
        $fresh[$entry.name] = $map
    }
    $fresh | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $Baseline
    Write-Host ("baseline written: {0} ({1} suites)" -f $Baseline, $fresh.Count)
}

Write-Host ("regression: {0} passed, {1} failed; results {2}" -f $report.passed, $report.failed, $ResultsPath)
if ($report.failed -gt 0) {
    foreach ($entry in ($results | Where-Object { $_.status -eq "fail" })) {
        Write-Host ("  {0}: {1}" -f $entry.name, ($entry.problems -join "; "))
    }
    exit 1
}