C64-dev/newLang/tests/regression/harness.tests.ps1

# E08-S01: harness self-tests. These prove the mechanics without starting an
# emulator: fresh build directories, a wall-clock limit, machine-readable
# results, artifact comparison, and a failed run that names the suite and its
# observed signature.
[CmdletBinding()]
param()

$ErrorActionPreference = "Stop"

$project = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent
$harness = Join-Path $PSScriptRoot "run-regression.ps1"
$root = Join-Path $project "build\regression-selftest"
$suites = Join-Path $root "suites"
$buildRoot = Join-Path $root "build"
$baseline = Join-Path $root "baseline.json"
$results = Join-Path $root "results.json"
$manifest = Join-Path $root "suites.ps1"

if (Test-Path -LiteralPath $root) { Remove-Item -LiteralPath $root -Recurse -Force }
New-Item -ItemType Directory -Force -Path $suites, $buildRoot | Out-Null

function Write-Suite([string]$Name, [string]$Body) {
    [IO.File]::WriteAllText((Join-Path $suites $Name), $Body,
        [Text.UTF8Encoding]::new($false))
}

# A passing suite writes one artifact and prints a stable success marker.
Write-Suite "ok.ps1" @'
$out = Join-Path (Split-Path $PSScriptRoot -Parent) "build\self-ok"
New-Item -ItemType Directory -Force -Path $out | Out-Null
[IO.File]::WriteAllBytes((Join-Path $out "self.prg"), [byte[]](1, 2, 3, 4))
Write-Host "verified self ok"
exit 0
'@
# A failing suite prints a diagnosis the harness must surface.
Write-Suite "bad.ps1" @'
Write-Host "self fixture ALPHA differs at token 7 (observed signature DEAD)"
exit 3
'@
# A hanging suite must be stopped by the wall-clock limit.
Write-Suite "slow.ps1" @'
Start-Sleep -Seconds 60
Write-Host "should never finish"
exit 0
'@
# A suite whose artifact content changes between runs.
Write-Suite "changed.ps1" @'
param()
$out = Join-Path (Split-Path $PSScriptRoot -Parent) "build\self-changed"
New-Item -ItemType Directory -Force -Path $out | Out-Null
$value = 5
if (Test-Path (Join-Path $out "bump.txt")) { $value = 6 }
[IO.File]::WriteAllText((Join-Path $out "bump.txt"), "$value")
[IO.File]::WriteAllBytes((Join-Path $out "self.err"), [byte[]]$value)
Write-Host "verified self changed"
exit 0
'@

[IO.File]::WriteAllText($manifest, @'
@(
    @{ Name = "ok"; Script = "build\regression-selftest\suites\ok.ps1"; BuildDir = "self-ok"; Timeout = 60; Success = "verified self ok"; Artifacts = @("*.prg") }
    @{ Name = "bad"; Script = "build\regression-selftest\suites\bad.ps1"; Timeout = 60 }
    @{ Name = "slow"; Script = "build\regression-selftest\suites\slow.ps1"; Timeout = 3 }
    @{ Name = "changed"; Script = "build\regression-selftest\suites\changed.ps1"; BuildDir = "self-changed"; Timeout = 60; Artifacts = @("*.err") }
)
'@, [Text.UTF8Encoding]::new($false))

function Invoke-Harness([string[]]$Extra) {
    $arguments = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", ('"' + $harness + '"'),
        "-Manifest", ('"' + $manifest + '"'), "-BuildRoot", ('"' + $buildRoot + '"'),
        "-Baseline", ('"' + $baseline + '"'), "-Results", ('"' + $results + '"')) + $Extra
    $output = & (Get-Command powershell).Source @arguments 2>&1 | Out-String
    return @{ Output = $output; Code = $LASTEXITCODE }
}

function Get-Report { Get-Content -Raw -LiteralPath $results | ConvertFrom-Json }
function Get-Entry([object]$Report, [string]$Name) {
    return $Report.suites | Where-Object { $_.name -eq $Name }
}

# 1: passing suite, artifact recorded, baseline written and matched on a rerun.
$first = Invoke-Harness @("-UpdateBaseline")
if (-not (Test-Path -LiteralPath $results)) { throw "first harness run wrote no results:`n$($first.Output)" }
$report = Get-Report
if ($report.passed -ne 2) { throw "expected 2 passing suites, got $($report.passed)" }
$ok = Get-Entry $report "ok"
if ($ok.artifacts.Count -ne 1 -or $ok.artifacts[0].status -ne "new") {
    throw "the passing suite did not record its artifact"
}
if (-not (Test-Path -LiteralPath $baseline)) { throw "-UpdateBaseline wrote no baseline" }
$second = Invoke-Harness @("-NoFresh")
$ok = Get-Entry (Get-Report) "ok"
if ($ok.artifacts[0].status -ne "match") {
    throw "a rerun did not match the recorded artifact:`n$($second.Output)"
}

# 2: a failing suite names the suite and the observed signature.
$bad = Get-Entry (Get-Report) "bad"
if ($bad.status -ne "fail" -or $bad.signature -notmatch "ALPHA differs") {
    throw "the failing suite did not report its signature: $($bad.signature)"
}
$slow = Get-Entry (Get-Report) "slow"
if ($slow.status -ne "fail" -or -not $slow.timedOut) {
    throw "the hanging suite was not stopped by the time limit"
}
if ($second.Code -eq 0) { throw "the harness reported success with failing suites" }

# 3: an artifact whose bytes change is reported as changed on the next run.
$third = Invoke-Harness @("-NoFresh")
$changed = Get-Entry (Get-Report) "changed"
if ($third.Code -eq 0 -or $changed.artifacts[0].status -ne "changed") {
    throw "a changed artifact was not detected:`n$($third.Output)"
}

# 4: results are machine-readable and identify the suite and its log.
if (-not $report.startedAt -or -not $report.finishedAt) { throw "results lack timestamps" }
foreach ($entry in $report.suites) {
    if (-not $entry.log -or -not (Test-Path -LiteralPath $entry.log)) {
        throw "suite $($entry.name) has no captured log"
    }
}

Write-Host "regression harness tests passed"
Write-Host "pass/fail/timeout/artifact-change detection, JSON results, captured logs"