mars-unicode-tables/scripts/check_nss_unitables.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-only
"""Parse and optionally compare NSS-compatible Unicode .TAB files.

This is a validation helper for TAB files generated by gen_nss_unitables.py.
It intentionally treats Novell NSS unitables only as local reference material:
the script parses their loader-visible structure and compares decoded mappings,
but does not copy table bytes or table data into generated output.
"""
from __future__ import annotations

import argparse
import struct
from dataclasses import dataclass
from pathlib import Path

UNI_UNDEFINED = 0xFFFD
CP_UNDEFINED = 0xFDFF
SINGLE_LOOKUP_TAG = 0x0F000000
MAPPING_TO_UPPER_CASE_BIT = 0x08000000
MAPPING_TO_LOWER_CASE_BIT = 0x04000000
OFFSET_MASK = 0x000FFFFF
UNI_000_SIZE = 0x10000 * 4 + 46507 * 4


@dataclass(frozen=True)
class ParsedUniTable:
    path: Path
    major: int
    minor: int
    tab_type: int
    size: int
    parsed_end: int
    mb_to_unicode: dict[int, int]
    unicode_to_mb: dict[int, int]
    lower: dict[int, int] | None = None
    upper: dict[int, int] | None = None


def _i32(data: bytes, offset: int) -> tuple[int, int]:
    return struct.unpack_from("<i", data, offset)[0], offset + 4


def _u32(data: bytes, offset: int) -> tuple[int, int]:
    return struct.unpack_from("<I", data, offset)[0], offset + 4


def _u16(data: bytes, offset: int) -> tuple[int, int]:
    return struct.unpack_from("<H", data, offset)[0], offset + 2


def _parse_version(header: bytes) -> tuple[int, int]:
    text = header.decode("latin-1", errors="replace")
    marker = "Version "
    idx = text.find(marker)
    if idx < 0:
        return 0, 0
    pos = idx + len(marker)
    end = pos
    while end < len(text) and (text[end].isdigit() or text[end] == "."):
        end += 1
    major_text, _, minor_text = text[pos:end].partition(".")
    try:
        return int(major_text), int(minor_text or "0")
    except ValueError:
        return 0, 0


def parse_uni_000(path: Path, data: bytes) -> ParsedUniTable:
    if len(data) != UNI_000_SIZE:
        raise ValueError(f"{path}: UNI_000.TAB size {len(data)} != expected {UNI_000_SIZE}")
    lower: dict[int, int] = {}
    upper: dict[int, int] = {}
    data_base = 0x10000 * 4
    data_count = (len(data) - data_base) // 4
    for ch in range(0x10000):
        lookup = struct.unpack_from("<I", data, ch * 4)[0]
        offset = lookup & OFFSET_MASK
        if lookup & MAPPING_TO_LOWER_CASE_BIT:
            if offset >= data_count:
                raise ValueError(f"{path}: lower offset out of range for U+{ch:04X}")
            lower[ch] = struct.unpack_from("<I", data, data_base + offset * 4)[0]
        if lookup & MAPPING_TO_UPPER_CASE_BIT:
            if lookup & MAPPING_TO_LOWER_CASE_BIT:
                offset += 1
            if offset >= data_count:
                raise ValueError(f"{path}: upper offset out of range for U+{ch:04X}")
            upper[ch] = struct.unpack_from("<I", data, data_base + offset * 4)[0]
    return ParsedUniTable(path, 0, 0, 0, len(data), len(data), {}, {}, lower, upper)


def parse_unitable(path: Path) -> ParsedUniTable:
    data = path.read_bytes()
    if path.name.upper() == "UNI_000.TAB":
        return parse_uni_000(path, data)
    if len(data) < 256 + 4:
        raise ValueError(f"{path}: too small for NSS unitable header")

    major, minor = _parse_version(data[:256])
    offset = 256
    tab_type, offset = _i32(data, offset)
    mb_to_unicode: dict[int, int] = {}

    if tab_type == 1:
        for code in range(256):
            unicode, offset = _u16(data, offset)
            if unicode not in (0, UNI_UNDEFINED):
                mb_to_unicode[code] = unicode
    elif tab_type == 2:
        cp_tab1: list[int] = []
        for _ in range(256):
            value, offset = _i32(data, offset)
            cp_tab1.append(value)
        min_lower, offset = _i32(data, offset)
        max_lower, offset = _i32(data, offset)
        tab2_count, offset = _i32(data, offset)
        cp_tab2: list[int] = []
        for _ in range(tab2_count):
            value, offset = _u16(data, offset)
            cp_tab2.append(value)

        for lead, index in enumerate(cp_tab1):
            if (index & 0xFF000000) == SINGLE_LOOKUP_TAG:
                unicode = index & 0xFFFF
                if unicode != UNI_UNDEFINED:
                    mb_to_unicode[lead] = unicode
                continue
            for low in range(min_lower, max_lower + 1):
                tab2_index = index + low
                if 0 <= tab2_index < len(cp_tab2):
                    unicode = cp_tab2[tab2_index]
                    if unicode != UNI_UNDEFINED:
                        mb_to_unicode[(lead << 8) | low] = unicode
    else:
        raise ValueError(f"{path}: unsupported tabType {tab_type}")

    uni_tab2_count, offset = _u32(data, offset)
    uni_tab1: list[int] = []
    for _ in range(256):
        value, offset = _u16(data, offset)
        uni_tab1.append(value)
    uni_tab2: list[int] = []
    for _ in range(uni_tab2_count):
        value, offset = _u16(data, offset)
        uni_tab2.append(value)

    unicode_to_mb: dict[int, int] = {}
    for unicode in range(0x10000):
        tab2_index = uni_tab1[unicode >> 8] + (unicode & 0xFF)
        if tab2_index < len(uni_tab2):
            code = uni_tab2[tab2_index]
            if code != CP_UNDEFINED:
                unicode_to_mb[unicode] = code

    return ParsedUniTable(
        path=path,
        major=major,
        minor=minor,
        tab_type=tab_type,
        size=len(data),
        parsed_end=offset,
        mb_to_unicode=mb_to_unicode,
        unicode_to_mb=unicode_to_mb,
    )


def _mapping_delta(reference: dict[int, int], generated: dict[int, int]) -> tuple[int, int, int, int, list[int]]:
    common = set(reference) & set(generated)
    diffs = [key for key in sorted(common) if reference[key] != generated[key]]
    return (
        len(common),
        len(set(reference) - set(generated)),
        len(set(generated) - set(reference)),
        len(diffs),
        diffs[:8],
    )


def print_summary(table: ParsedUniTable) -> None:
    if table.path.name.upper() == "UNI_000.TAB":
        print(
            f"{table.path}: UNI_000 bytes={table.size} parsed_end={table.parsed_end} "
            f"lower={len(table.lower or {})} upper={len(table.upper or {})}"
        )
        return
    print(
        f"{table.path}: version={table.major}.{table.minor} tabType={table.tab_type} "
        f"bytes={table.size} parsed_end={table.parsed_end} "
        f"mb2uni={len(table.mb_to_unicode)} uni2mb={len(table.unicode_to_mb)}"
    )


def compare(reference: ParsedUniTable, generated: ParsedUniTable) -> int:
    print_summary(reference)
    print_summary(generated)
    if reference.path.name.upper() == "UNI_000.TAB" or generated.path.name.upper() == "UNI_000.TAB":
        lower_common, lower_ref_only, lower_gen_only, lower_diff_count, lower_diffs = _mapping_delta(
            reference.lower or {}, generated.lower or {}
        )
        upper_common, upper_ref_only, upper_gen_only, upper_diff_count, upper_diffs = _mapping_delta(
            reference.upper or {}, generated.upper or {}
        )
        print(
            f"  lower: common={lower_common} reference_only={lower_ref_only} "
            f"generated_only={lower_gen_only} differing={lower_diff_count} sample={lower_diffs}"
        )
        print(
            f"  upper: common={upper_common} reference_only={upper_ref_only} "
            f"generated_only={upper_gen_only} differing={upper_diff_count} sample={upper_diffs}"
        )
        return 1 if lower_diff_count or upper_diff_count else 0
    mb_common, mb_ref_only, mb_gen_only, mb_diff_count, mb_diffs = _mapping_delta(
        reference.mb_to_unicode, generated.mb_to_unicode
    )
    uni_common, uni_ref_only, uni_gen_only, uni_diff_count, uni_diffs = _mapping_delta(
        reference.unicode_to_mb, generated.unicode_to_mb
    )
    print(
        f"  mb2uni: common={mb_common} reference_only={mb_ref_only} "
        f"generated_only={mb_gen_only} differing={mb_diff_count} sample={mb_diffs}"
    )
    print(
        f"  uni2mb: common={uni_common} reference_only={uni_ref_only} "
        f"generated_only={uni_gen_only} differing={uni_diff_count} sample={uni_diffs}"
    )
    return 1 if mb_diff_count or uni_diff_count else 0


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("tables", nargs="*", type=Path, help="Generated .TAB files to parse")
    parser.add_argument(
        "--reference-dir",
        type=Path,
        help="Optional Novell NSS shared/sdk/unitables directory for local comparison",
    )
    args = parser.parse_args()

    status = 0
    for table_path in args.tables:
        generated = parse_unitable(table_path)
        if args.reference_dir:
            reference_path = args.reference_dir / table_path.name
            if reference_path.exists():
                status |= compare(parse_unitable(reference_path), generated)
            else:
                print_summary(generated)
                print(f"  no reference table: {reference_path}")
        else:
            print_summary(generated)
    raise SystemExit(status)


if __name__ == "__main__":
    main()