dcovington
/
C64-dev


			
				
					
						
						
							
							# Host reference lexer for Nova128 (E02-S02). Dot-source this file.
#
# An independent implementation of docs/language-spec.md 2.2 plus the token
# rules documented at the top of src/compiler/lexer.s. Golden files must be
# reproduced by BOTH this lexer and the C128 lexer.
#
# Token text format, one per line:
#   L:C KIND [raw-length] [detail]
#   IDENT    detail = normalized name (PETSCII $41-$5A shown as A-Z)
#   NUMBER   detail = decimal value
#   STRING   detail = "text" (see Format-PetsciiText)
#   CHAR     detail = $hh
#   ERROR    detail = LEXERR name (no raw length)
#   EOL/EOF  no length

$script:Keywords = @{
    "AND" = "KW_AND"; "AS" = "KW_AS"; "BOOLEAN" = "KW_BOOLEAN"; "BYTE" = "KW_BYTE"
    "CHAR" = "KW_CHAR"; "CLASS" = "KW_CLASS"; "DELETE" = "KW_DELETE"; "ELSE" = "KW_ELSE"
    "END" = "KW_END"; "FALSE" = "KW_FALSE"; "FOR" = "KW_FOR"; "IF" = "KW_IF"
    "IMPLEMENTS" = "KW_IMPLEMENTS"; "INTEGER" = "KW_INTEGER"; "INTERFACE" = "KW_INTERFACE"
    "METHOD" = "KW_METHOD"; "NEW" = "KW_NEW"; "NOT" = "KW_NOT"; "NULL" = "KW_NULL"
    "OR" = "KW_OR"; "PRINT" = "KW_PRINT"; "PRIVATE" = "KW_PRIVATE"; "PROGRAM" = "KW_PROGRAM"
    "PUBLIC" = "KW_PUBLIC"; "RETURN" = "KW_RETURN"; "SELF" = "KW_SELF"; "STEP" = "KW_STEP"
    "TO" = "KW_TO"; "TRUE" = "KW_TRUE"; "VAR" = "KW_VAR"; "WHILE" = "KW_WHILE"; "WORD" = "KW_WORD"
}

# Token kind numbers from src/common/tokens.inc (used to decode C128 dumps).
$script:KindNames = @{
    0x00 = "EOF"; 0x01 = "EOL"; 0x02 = "IDENT"; 0x03 = "NUMBER"; 0x04 = "STRING"
    0x05 = "CHAR"; 0x06 = "ERROR"
    0x10 = "LPAREN"; 0x11 = "RPAREN"; 0x12 = "LBRACKET"; 0x13 = "RBRACKET"; 0x14 = "COMMA"
    0x15 = "DOT"; 0x16 = "COLON"; 0x17 = "EQ"; 0x18 = "PLUS"; 0x19 = "MINUS"; 0x1A = "STAR"
    0x1B = "SLASH"; 0x1C = "LT"; 0x1D = "GT"; 0x1E = "NE"; 0x1F = "LE"; 0x20 = "GE"
}
$i = 0x40
foreach ($kw in ($script:Keywords.Keys | Sort-Object)) {
    $script:KindNames[$i] = $script:Keywords[$kw]; $i++
}
$script:ErrorNames = @{
    1 = "INVALID_CHAR"; 2 = "IDENT_TOO_LONG"; 3 = "NUMBER_RANGE"; 4 = "BAD_NUMBER"
    5 = "UNTERMINATED_STRING"; 6 = "BAD_ESCAPE"; 7 = "BAD_CHAR_LITERAL"
}
$script:Punct = @{
    0x28 = "LPAREN"; 0x29 = "RPAREN"; 0x5B = "LBRACKET"; 0x5D = "RBRACKET"; 0x2C = "COMMA"
    0x2E = "DOT"; 0x3A = "COLON"; 0x3D = "EQ"; 0x2B = "PLUS"; 0x2D = "MINUS"; 0x2A = "STAR"
    0x2F = "SLASH"; 0x3C = "LT"; 0x3E = "GT"
}

function Format-PetsciiText([byte[]]$bytes) {
    $sb = New-Object System.Text.StringBuilder
    foreach ($b in $bytes) {
        if ($b -ge 0x41 -and $b -le 0x5A) { [void]$sb.Append([char]($b + 0x20)) }      # a-z
        elseif ($b -ge 0xC1 -and $b -le 0xDA) { [void]$sb.Append([char]($b - 0x80)) } # A-Z
        elseif ($b -ge 0x20 -and $b -le 0x3F -and $b -ne 0x22) { [void]$sb.Append([char]$b) }
        else { [void]$sb.Append(("{{{0:X2}}}" -f $b)) }
    }
    return '"' + $sb.ToString() + '"'
}

function Format-Name([byte[]]$bytes) {
    return -join ($bytes | ForEach-Object { [char]$_ })   # $41-$5A, digits, $5F
}

function Test-Letter([int]$b) { return ($b -ge 0x41 -and $b -le 0x5A) -or ($b -ge 0xC1 -and $b -le 0xDA) }
function Test-Digit([int]$b) { return $b -ge 0x30 -and $b -le 0x39 }
function Test-IdentPart([int]$b) { return (Test-Letter $b) -or (Test-Digit $b) -or $b -eq 0x5F }

# Returns the token lines for one PETSCII source.
function Get-ReferenceTokens([byte[]]$Source) {
    $out = New-Object System.Collections.Generic.List[string]
    $lines = New-Object System.Collections.Generic.List[byte[]]
    $start = 0
    for ($k = 0; $k -lt $Source.Length; $k++) {
        if ($Source[$k] -eq 13) {
            $lines.Add([byte[]]$(if ($k -gt $start) { $Source[$start..($k - 1)] } else { @() }))
            $start = $k + 1
        }
    }
    if ($start -lt $Source.Length) { $lines.Add([byte[]]$Source[$start..($Source.Length - 1)]) }

    for ($ln = 0; $ln -lt $lines.Count; $ln++) {
        $line = $lines[$ln]; $n = $line.Length; $lineNo = $ln + 1
        $at = { param($p) if ($p -lt $n) { [int]$line[$p] } else { 13 } }
        $p = 0
        while ($true) {
            while ((& $at $p) -eq 0x20 -or (& $at $p) -eq 0x09) { $p++ }
            $c = & $at $p
            if ($c -eq 13) { break }
            $col = $p + 1; $s = $p
            $emit = { param($kind, $detail) $out.Add(("{0}:{1} {2} {3}{4}" -f $lineNo, $col, $kind, ($p - $s), $detail)) }
            $err = $null; $errCol = $col

            if (Test-Letter $c) {
                $name = New-Object System.Collections.Generic.List[byte]
                while (Test-IdentPart (& $at $p)) {
                    $b = & $at $p
                    if (Test-Letter $b) { $b = $b -band 0x7F }
                    $name.Add([byte]$b); $p++
                }
                if ($name.Count -gt 16) { $err = "IDENT_TOO_LONG" }
                else {
                    $text = Format-Name $name.ToArray()
                    if ($script:Keywords.ContainsKey($text)) { & $emit $script:Keywords[$text] "" }
                    else { & $emit "IDENT" (" " + $text) }
                }
            } elseif ((Test-Digit $c) -or $c -eq 0x24 -or $c -eq 0x25) {
                $value = [long]0; $count = 0
                if ($c -eq 0x24) {
                    $p++
                    while ($true) {
                        $b = & $at $p; $d = -1
                        if (Test-Digit $b) { $d = $b - 0x30 }
                        elseif ((Test-Letter $b) -and ($b -band 0x7F) -ge 0x41 -and ($b -band 0x7F) -le 0x46) {
                            $d = ($b -band 0x7F) - 0x41 + 10
                        }
                        if ($d -lt 0) { break }
                        $value = $value * 16 + $d; $count++; $p++
                    }
                } elseif ($c -eq 0x25) {
                    $p++
                    while ((& $at $p) -eq 0x30 -or (& $at $p) -eq 0x31) {
                        $value = $value * 2 + ((& $at $p) - 0x30); $count++; $p++
                    }
                } else {
                    while (Test-Digit (& $at $p)) { $value = $value * 10 + ((& $at $p) - 0x30); $count++; $p++ }
                }
                if ($count -eq 0 -or (Test-IdentPart (& $at $p))) { $err = "BAD_NUMBER" }
                elseif ($value -gt 65535) { $err = "NUMBER_RANGE" }
                else { & $emit "NUMBER" (" " + $value) }
            } elseif ($c -eq 0x22) {
                $p++; $text = New-Object System.Collections.Generic.List[byte]
                while ($null -eq $err) {
                    $b = & $at $p
                    if ($b -eq 13) { $err = "UNTERMINATED_STRING"; break }
                    if ($b -eq 0x22) { $p++; break }
                    if ($b -eq 0x5C) {
                        $e = & $at ($p + 1)
                        if ($e -eq 0x5C -or $e -eq 0x22) { $b = $e }
                        elseif (($e -band 0x7F) -eq 0x4E -or ($e -band 0x7F) -eq 0x52) { $b = 0x0D }
                        else { $err = "BAD_ESCAPE"; $errCol = $p + 1; break }
                        $p++
                    }
                    $text.Add([byte]$b); $p++
                }
                if ($null -eq $err) { & $emit "STRING" (" " + (Format-PetsciiText $text.ToArray())) }
            } elseif ($c -eq 0x27) {
                $p++; $b = & $at $p
                if ($b -eq 13 -or $b -eq 0x27) { $err = "BAD_CHAR_LITERAL" }
                else {
                    if ($b -eq 0x5C) {
                        $e = & $at ($p + 1)
                        if ($e -eq 0x5C -or $e -eq 0x27) { $b = $e }
                        elseif (($e -band 0x7F) -eq 0x4E -or ($e -band 0x7F) -eq 0x52) { $b = 0x0D }
                        else { $err = "BAD_ESCAPE"; $errCol = $p + 1 }
                        $p++
                    }
                    if ($null -eq $err) {
                        $p++
                        if ((& $at $p) -ne 0x27) { $err = "BAD_CHAR_LITERAL" }
                        else { $p++; & $emit "CHAR" (" `${0:X2}" -f $b) }
                    }
                }
            } elseif ($script:Punct.ContainsKey($c)) {
                $next = & $at ($p + 1)
                if ($c -eq 0x2F -and $next -eq 0x2F) { break }               # comment
                $p++
                if ($c -eq 0x3C -and $next -eq 0x3E) { $p++; & $emit "NE" "" }
                elseif ($c -eq 0x3C -and $next -eq 0x3D) { $p++; & $emit "LE" "" }
                elseif ($c -eq 0x3E -and $next -eq 0x3D) { $p++; & $emit "GE" "" }
                else { & $emit $script:Punct[$c] "" }
            } else {
                $err = "INVALID_CHAR"
            }

            if ($null -ne $err) {
                $out.Add(("{0}:{1} ERROR {2}" -f $lineNo, $errCol, $err))
                break                                                     # skip the line
            }
        }
        $out.Add(("{0}:{1} EOL" -f $lineNo, ($n + 1)))
    }
    $out.Add(("{0}:0 EOF" -f $lines.Count))
    return ,$out
}

# Decode the C128 dump (tests/lexer/lexer-test.s) into per-fixture token lines.
# Returns a list (in dump order) of @{ Index; Load; Tokens }.
function ConvertFrom-TokenDump([byte[]]$Dump) {
    $result = New-Object System.Collections.Generic.List[object]
    $p = 0; $current = $null
    while ($p -lt $Dump.Length) {
        $kind = $Dump[$p]
        if ($kind -eq 0xFE) { break }
        if ($kind -eq 0xFF) {
            $current = [pscustomobject]@{ Index = [int]$Dump[$p + 1]; Load = $Dump[$p + 2]
                Tokens = New-Object System.Collections.Generic.List[string] }
            $result.Add($current)
            $p += 3; continue
        }
        $line = $Dump[$p + 1] + 256 * $Dump[$p + 2]; $col = $Dump[$p + 3]; $p += 4
        $name = $script:KindNames[[int]$kind]
        if ($null -eq $name) { throw ("unknown token kind `${0:X2} in dump" -f $kind) }
        switch ($name) {
            "EOF" { $current.Tokens.Add("${line}:$col EOF") }
            "EOL" { $current.Tokens.Add("${line}:$col EOL") }
            "ERROR" { $current.Tokens.Add("${line}:$col ERROR " + $script:ErrorNames[[int]$Dump[$p]]); $p++ }
            { $_ -in "IDENT", "STRING" } {
                $len = $Dump[$p]; $tl = $Dump[$p + 1]
                $text = if ($tl -gt 0) { [byte[]]$Dump[($p + 2)..($p + 1 + $tl)] } else { [byte[]]@() }
                $detail = if ($name -eq "IDENT") { Format-Name $text } else { Format-PetsciiText $text }
                $current.Tokens.Add("${line}:$col $name $len $detail"); $p += 2 + $tl
            }
            "NUMBER" { $current.Tokens.Add("${line}:$col NUMBER $($Dump[$p]) $($Dump[$p + 1] + 256 * $Dump[$p + 2])"); $p += 3 }
            "CHAR" { $current.Tokens.Add(("{0}:{1} CHAR {2} `${3:X2}" -f $line, $col, $Dump[$p], $Dump[$p + 1])); $p += 2 }
            default { $current.Tokens.Add("${line}:$col $name $($Dump[$p])"); $p++ }
        }
    }
    return ,$result
}