#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """Parse and optionally compare NSS-compatible Unicode .TAB files. This is a validation helper for TAB files generated by gen_nss_unitables.py. It intentionally treats Novell NSS unitables only as local reference material: the script parses their loader-visible structure and compares decoded mappings, but does not copy table bytes or table data into generated output. """ from __future__ import annotations import argparse import struct from dataclasses import dataclass from pathlib import Path UNI_UNDEFINED = 0xFFFD CP_UNDEFINED = 0xFDFF SINGLE_LOOKUP_TAG = 0x0F000000 MAPPING_TO_UPPER_CASE_BIT = 0x08000000 MAPPING_TO_LOWER_CASE_BIT = 0x04000000 OFFSET_MASK = 0x000FFFFF UNI_000_SIZE = 0x10000 * 4 + 46507 * 4 @dataclass(frozen=True) class ParsedUniTable: path: Path major: int minor: int tab_type: int size: int parsed_end: int mb_to_unicode: dict[int, int] unicode_to_mb: dict[int, int] lower: dict[int, int] | None = None upper: dict[int, int] | None = None def _i32(data: bytes, offset: int) -> tuple[int, int]: return struct.unpack_from(" tuple[int, int]: return struct.unpack_from(" tuple[int, int]: return struct.unpack_from(" tuple[int, int]: text = header.decode("latin-1", errors="replace") marker = "Version " idx = text.find(marker) if idx < 0: return 0, 0 pos = idx + len(marker) end = pos while end < len(text) and (text[end].isdigit() or text[end] == "."): end += 1 major_text, _, minor_text = text[pos:end].partition(".") try: return int(major_text), int(minor_text or "0") except ValueError: return 0, 0 def parse_uni_000(path: Path, data: bytes) -> ParsedUniTable: if len(data) != UNI_000_SIZE: raise ValueError(f"{path}: UNI_000.TAB size {len(data)} != expected {UNI_000_SIZE}") lower: dict[int, int] = {} upper: dict[int, int] = {} data_base = 0x10000 * 4 data_count = (len(data) - data_base) // 4 for ch in range(0x10000): lookup = struct.unpack_from("= data_count: raise ValueError(f"{path}: lower offset out of range for U+{ch:04X}") lower[ch] = struct.unpack_from("= data_count: raise ValueError(f"{path}: upper offset out of range for U+{ch:04X}") upper[ch] = struct.unpack_from(" ParsedUniTable: data = path.read_bytes() if path.name.upper() == "UNI_000.TAB": return parse_uni_000(path, data) if len(data) < 256 + 4: raise ValueError(f"{path}: too small for NSS unitable header") major, minor = _parse_version(data[:256]) offset = 256 tab_type, offset = _i32(data, offset) mb_to_unicode: dict[int, int] = {} if tab_type == 1: for code in range(256): unicode, offset = _u16(data, offset) if unicode not in (0, UNI_UNDEFINED): mb_to_unicode[code] = unicode elif tab_type == 2: cp_tab1: list[int] = [] for _ in range(256): value, offset = _i32(data, offset) cp_tab1.append(value) min_lower, offset = _i32(data, offset) max_lower, offset = _i32(data, offset) tab2_count, offset = _i32(data, offset) cp_tab2: list[int] = [] for _ in range(tab2_count): value, offset = _u16(data, offset) cp_tab2.append(value) for lead, index in enumerate(cp_tab1): if (index & 0xFF000000) == SINGLE_LOOKUP_TAG: unicode = index & 0xFFFF if unicode != UNI_UNDEFINED: mb_to_unicode[lead] = unicode continue for low in range(min_lower, max_lower + 1): tab2_index = index + low if 0 <= tab2_index < len(cp_tab2): unicode = cp_tab2[tab2_index] if unicode != UNI_UNDEFINED: mb_to_unicode[(lead << 8) | low] = unicode else: raise ValueError(f"{path}: unsupported tabType {tab_type}") uni_tab2_count, offset = _u32(data, offset) uni_tab1: list[int] = [] for _ in range(256): value, offset = _u16(data, offset) uni_tab1.append(value) uni_tab2: list[int] = [] for _ in range(uni_tab2_count): value, offset = _u16(data, offset) uni_tab2.append(value) unicode_to_mb: dict[int, int] = {} for unicode in range(0x10000): tab2_index = uni_tab1[unicode >> 8] + (unicode & 0xFF) if tab2_index < len(uni_tab2): code = uni_tab2[tab2_index] if code != CP_UNDEFINED: unicode_to_mb[unicode] = code return ParsedUniTable( path=path, major=major, minor=minor, tab_type=tab_type, size=len(data), parsed_end=offset, mb_to_unicode=mb_to_unicode, unicode_to_mb=unicode_to_mb, ) def _mapping_delta(reference: dict[int, int], generated: dict[int, int]) -> tuple[int, int, int, int, list[int]]: common = set(reference) & set(generated) diffs = [key for key in sorted(common) if reference[key] != generated[key]] return ( len(common), len(set(reference) - set(generated)), len(set(generated) - set(reference)), len(diffs), diffs[:8], ) def print_summary(table: ParsedUniTable) -> None: if table.path.name.upper() == "UNI_000.TAB": print( f"{table.path}: UNI_000 bytes={table.size} parsed_end={table.parsed_end} " f"lower={len(table.lower or {})} upper={len(table.upper or {})}" ) return print( f"{table.path}: version={table.major}.{table.minor} tabType={table.tab_type} " f"bytes={table.size} parsed_end={table.parsed_end} " f"mb2uni={len(table.mb_to_unicode)} uni2mb={len(table.unicode_to_mb)}" ) def compare(reference: ParsedUniTable, generated: ParsedUniTable) -> int: print_summary(reference) print_summary(generated) if reference.path.name.upper() == "UNI_000.TAB" or generated.path.name.upper() == "UNI_000.TAB": lower_common, lower_ref_only, lower_gen_only, lower_diff_count, lower_diffs = _mapping_delta( reference.lower or {}, generated.lower or {} ) upper_common, upper_ref_only, upper_gen_only, upper_diff_count, upper_diffs = _mapping_delta( reference.upper or {}, generated.upper or {} ) print( f" lower: common={lower_common} reference_only={lower_ref_only} " f"generated_only={lower_gen_only} differing={lower_diff_count} sample={lower_diffs}" ) print( f" upper: common={upper_common} reference_only={upper_ref_only} " f"generated_only={upper_gen_only} differing={upper_diff_count} sample={upper_diffs}" ) return 1 if lower_diff_count or upper_diff_count else 0 mb_common, mb_ref_only, mb_gen_only, mb_diff_count, mb_diffs = _mapping_delta( reference.mb_to_unicode, generated.mb_to_unicode ) uni_common, uni_ref_only, uni_gen_only, uni_diff_count, uni_diffs = _mapping_delta( reference.unicode_to_mb, generated.unicode_to_mb ) print( f" mb2uni: common={mb_common} reference_only={mb_ref_only} " f"generated_only={mb_gen_only} differing={mb_diff_count} sample={mb_diffs}" ) print( f" uni2mb: common={uni_common} reference_only={uni_ref_only} " f"generated_only={uni_gen_only} differing={uni_diff_count} sample={uni_diffs}" ) return 1 if mb_diff_count or uni_diff_count else 0 def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("tables", nargs="*", type=Path, help="Generated .TAB files to parse") parser.add_argument( "--reference-dir", type=Path, help="Optional Novell NSS shared/sdk/unitables directory for local comparison", ) args = parser.parse_args() status = 0 for table_path in args.tables: generated = parse_unitable(table_path) if args.reference_dir: reference_path = args.reference_dir / table_path.name if reference_path.exists(): status |= compare(parse_unitable(reference_path), generated) else: print_summary(generated) print(f" no reference table: {reference_path}") else: print_summary(generated) raise SystemExit(status) if __name__ == "__main__": main()