Stirling-PDF/scripts/sync_en_us_spelling.py

#!/usr/bin/env python3
"""Sync en-US translations from en-GB and normalise spelling variants.

Two jobs, one pass:

1. Copy every key that exists in en-GB but is missing from en-US into en-US,
   converting British spellings to American on the way in. Keys that only exist
   in en-US (e.g. SaaS-only strings) are left untouched.

2. Fix any American spellings that have leaked into the en-GB *values* by
   converting them back to British.

Rules that keep this safe:
  * Only the VALUE (the quoted right-hand side) is ever rewritten. TOML keys and
    section headers are never touched -- they are code identifiers.
  * Matching is whole-word and case-preserving, driven by an explicit word map.
    We never match on substrings/stems, so "parameters", "entire", "literal",
    "programmatically", "checkbox" etc. are never mangled.
  * Software-ambiguous words whose "British" form would be wrong in a UI
    (program, dialog, disk, check/cheque, story, catalog as a CS term...) are
    deliberately left out of the map. See AMBIGUOUS_NOTES.

Usage:
    python3 scripts/sync_en_us_spelling.py            # apply changes
    python3 scripts/sync_en_us_spelling.py --dry-run   # report only
"""

from __future__ import annotations

import argparse
import re
import sys
from pathlib import Path

LOCALES = Path(__file__).resolve().parent.parent / "frontend/editor/public/locales"
EN_US = LOCALES / "en-US/translation.toml"
EN_GB = LOCALES / "en-GB/translation.toml"

# --------------------------------------------------------------------------- #
# Spelling map: British (uk) -> American (us), as full whole words.
#
# Inflected forms are generated for the regular families so the map stays
# readable. Irregular / one-off pairs are listed explicitly at the bottom.
# --------------------------------------------------------------------------- #

# Words intentionally NOT mapped because the "British" form is wrong or
# ambiguous in a software UI context (documented, not applied):
AMBIGUOUS_NOTES = {
    "program/programme": "‘program’ is correct in British English for software.",
    "dialog/dialogue": "‘dialog’ is standard UI terminology (dialog box) in all locales.",
    "disk/disc": "‘disk’ is standard for storage in all locales.",
    "check/cheque": "‘check’ (verb/checkbox) is identical in both; only the money sense differs.",
    "analog/analogue": "rare in this UI and collides with technical usage.",
    "story/storey": "‘story’ (narrative/UI) is identical; only the building-floor sense differs.",
    "meter/metre": "‘meter’ (a device/‘metered’) is valid in British English too.",
}

uk_to_us: dict[str, str] = {}


def add(uk: str, us: str) -> None:
    """Register a British->American whole-word pair (lower-cased key)."""
    uk_to_us[uk.lower()] = us.lower()


# --- -our / -or family (colour -> color) ---------------------------------- #
# us = uk with the single 'our' turned into 'or'.
_OUR = [
    "colour", "colours", "coloured", "colouring", "colourful", "colourless",
    "colourise", "colourised", "colouriser",
    "favour", "favours", "favoured", "favouring", "favourite", "favourites",
    "favourable", "favourably", "favouritism",
    "behaviour", "behaviours", "behavioural",
    "neighbour", "neighbours", "neighbouring",
    "labour", "labours", "laboured", "labouring",
    "honour", "honours", "honoured", "honouring", "honourable",
    "humour", "humours", "humoured",
    "flavour", "flavours", "flavoured", "flavouring",
    "harbour", "harbours",
    "rumour", "rumours", "rumoured",
    "vapour", "vapours",
    "odour", "odours",
    "valour", "splendour", "armour", "armoured",
    "saviour", "saviours",
    "endeavour", "endeavours", "endeavoured",
    "parlour", "savour", "savoured", "savouring", "savoury",
    "candour", "demeanour", "rigour", "vigour",
]
for w in _OUR:
    add(w, w.replace("our", "or"))

# --- -re / -er family (centre -> center) ---------------------------------- #
_RE = {
    "centre": "center", "centres": "centers", "centred": "centered",
    "centring": "centering",
    "metre": "meter", "metres": "meters",          # length unit; ‘meter’ device excluded above
    "litre": "liter", "litres": "liters",
    "fibre": "fiber", "fibres": "fibers",
    # NB: ‘calibre’ is omitted -- in this codebase it is the proper noun
    # ‘Calibre’ (the e-book conversion tool), not the British ‘caliber’.
    "sombre": "somber", "spectre": "specter", "lustre": "luster",
    "theatre": "theater", "theatres": "theaters",
    "metre": "meter",
    "centimetre": "centimeter", "centimetres": "centimeters",
    "millimetre": "millimeter", "millimetres": "millimeters",
    "kilometre": "kilometer", "kilometres": "kilometers",
    "manoeuvre": "maneuver", "manoeuvres": "maneuvers",
}
for uk, us in _RE.items():
    add(uk, us)

# --- -ise / -isation family (organise -> organize) ------------------------ #
# Generate the regular inflections from a list of stems (the part before 'se').
_ISE_STEMS = [
    "organi", "recogni", "customi", "optimi", "authori", "reali", "finali",
    "initiali", "normali", "synchroni", "summari", "prioriti", "personali",
    "capitali", "categori", "standardi", "visuali", "apologi", "utili",
    "locali", "digiti", "moderni", "centrali", "minimi", "maximi", "emphasi",
    "characteri", "stabili", "generali", "specifi? ".strip(),
    "sterili", "neutrali", "saniti", "tokeni", "serie? ".strip(),
    "alphabeti", "synthesi", "memori", "itemi", "colouri",
]
_ISE_STEMS = [s for s in _ISE_STEMS if s and not s.endswith("?")]
_ISE_SUFFIXES = ["se", "ses", "sed", "sing", "ser", "sers", "sation", "sations"]
for stem in _ISE_STEMS:
    for suf in _ISE_SUFFIXES:
        add(stem + suf, stem + suf.replace("s", "z", 1))

# -yse -> -yze verbs. Deliberately excludes the 3rd-person ‘analyses/analyzes’
# because it collides with the noun ‘analyses’ (identical in both locales), and
# ‘analysis/analyses’ which are spelled the same on both sides.
for uk, us in {
    "analyse": "analyze", "analysed": "analyzed", "analysing": "analyzing",
    "analyser": "analyzer",
    "paralyse": "paralyze", "paralysed": "paralyzed", "paralysing": "paralyzing",
    "catalyse": "catalyze", "catalysed": "catalyzed",
}.items():
    add(uk, us)

# --- doubled-l family (cancelled -> canceled) ----------------------------- #
_LL = {
    "cancelled": "canceled", "cancelling": "canceling",
    "labelled": "labeled", "labelling": "labeling",
    "modelled": "modeled", "modelling": "modeling",
    "signalled": "signaled", "signalling": "signaling",
    "travelled": "traveled", "travelling": "traveling", "traveller": "traveler",
    "travellers": "travelers",
    "fuelled": "fueled", "fuelling": "fueling",
    "marvellous": "marvelous", "counsellor": "counselor",
    "jewellery": "jewelry",
}
for uk, us in _LL.items():
    add(uk, us)

# single-l where US doubles it (enrol -> enroll, fulfil -> fulfill)
_L_TO_LL = {
    "enrol": "enroll", "enrols": "enrolls", "enrolment": "enrollment",
    "enrolments": "enrollments",
    "fulfil": "fulfill", "fulfils": "fulfills", "fulfilment": "fulfillment",
    "fulfilments": "fulfillments",
    "instalment": "installment", "instalments": "installments",
    "skilful": "skillful", "wilful": "willful",
}
for uk, us in _L_TO_LL.items():
    add(uk, us)

# --- -ce / -se nouns (licence -> license, defence -> defense) ------------- #
# Only the unambiguous noun forms; verb/adjective forms ‘licensed/licensing’
# are identical in both and are left alone.
_CE = {
    "licence": "license", "licences": "licenses",
    "defence": "defense", "defences": "defenses",
    "offence": "offense", "offences": "offenses",
    "pretence": "pretense",
}
for uk, us in _CE.items():
    add(uk, us)

# --- -ogue (catalogue -> catalog) ----------------------------------------- #
# Note: ‘dialogue/dialog’ and ‘analogue/analog’ are excluded (UI terminology).
_OGUE = {
    "catalogue": "catalog", "catalogues": "catalogs",
    "catalogued": "cataloged", "cataloguing": "cataloging",
}
for uk, us in _OGUE.items():
    add(uk, us)

# --- miscellaneous irregulars --------------------------------------------- #
_MISC = {
    "grey": "gray", "greys": "grays", "greyed": "grayed", "greying": "graying",
    "greyscale": "grayscale",
    "mould": "mold", "moulds": "molds",
    "sceptical": "skeptical", "sceptic": "skeptic", "scepticism": "skepticism",
    "aluminium": "aluminum",
    "artefact": "artifact", "artefacts": "artifacts",
    "speciality": "specialty", "specialities": "specialties",
    "judgement": "judgment", "judgements": "judgments",
    "acknowledgement": "acknowledgment", "acknowledgements": "acknowledgments",
    "kerb": "curb", "kerbs": "curbs",
    "tyre": "tire", "tyres": "tires",
    "sulphur": "sulfur",
    "enquiry": "inquiry", "enquiries": "inquiries",
    "ageing": "aging",
    "cancelled": "canceled",  # belt & braces (also in _LL)
    # NB: while/whilst, toward/towards, among/amongst are deliberately omitted.
    # They are lexical/register choices, not spelling variants -- and "while",
    # "toward", "among" are all valid British English. Converting them would be
    # intrusive and out of scope for a spelling pass.
}
for uk, us in _MISC.items():
    add(uk, us)

# Reverse map for fixing American spellings inside en-GB.
us_to_uk: dict[str, str] = {}
for uk, us in uk_to_us.items():
    # Don't overwrite a genuine British form that happens to equal another US form.
    us_to_uk.setdefault(us, uk)

# --------------------------------------------------------------------------- #
# Case-preserving whole-word replacement.
# --------------------------------------------------------------------------- #

def _match_case(template: str, replacement: str) -> str:
    if template.isupper():
        return replacement.upper()
    if template[:1].isupper():
        return replacement[:1].upper() + replacement[1:]
    return replacement


# Literal technical tokens that must never be respelled, even though they look
# like convertible words. These are protocol/identifier strings, not prose.
PROTECTED = re.compile(
    r"Authorization(?=\s*:)"        # the HTTP "Authorization:" header
    r"|Authorization\s+header",
    re.IGNORECASE,
)
_SENTINEL = "\x00{}\x00"


def make_converter(mapping: dict[str, str]):
    if not mapping:
        return lambda text: (text, [])
    # Longest-first so multi-word/longer forms win; \b ensures whole words.
    pattern = re.compile(
        r"\b(" + "|".join(re.escape(w) for w in sorted(mapping, key=len, reverse=True)) + r")\b",
        re.IGNORECASE,
    )

    def convert(text: str) -> tuple[str, list[tuple[str, str]]]:
        # Mask protected literals so they pass through untouched.
        protected: list[str] = []

        def mask(m: re.Match[str]) -> str:
            protected.append(m.group(0))
            return _SENTINEL.format(len(protected) - 1)

        masked = PROTECTED.sub(mask, text)

        changes: list[tuple[str, str]] = []

        def repl(m: re.Match[str]) -> str:
            src = m.group(0)
            dst = _match_case(src, mapping[src.lower()])
            if dst != src:
                changes.append((src, dst))
            return dst

        converted = pattern.sub(repl, masked)
        for i, original in enumerate(protected):
            converted = converted.replace(_SENTINEL.format(i), original)
        return converted, changes

    return convert


uk_to_us_convert = make_converter(uk_to_us)
us_to_uk_convert = make_converter(us_to_uk)

# --------------------------------------------------------------------------- #
# TOML line model (value-only edits; structure preserved verbatim).
# --------------------------------------------------------------------------- #

SECTION_RE = re.compile(r"^\[(.+)\]$")
KV_RE = re.compile(r'^([A-Za-z0-9_.-]+)\s*=\s*"(.*)"$')


def fq(section: str, key: str) -> str:
    return f"{section}.{key}" if section else key


def parse_keys(path: Path) -> tuple[dict[str, str], list[tuple[str, str, str]]]:
    """Return {fq_key: value} and an ordered list of (section, key, value)."""
    keys: dict[str, str] = {}
    ordered: list[tuple[str, str, str]] = []
    section = ""
    for raw in path.read_text(encoding="utf-8").splitlines():
        s = raw.strip()
        if not s or s.startswith("#"):
            continue
        sec = SECTION_RE.match(s)
        if sec:
            section = sec.group(1)
            continue
        kv = KV_RE.match(s)
        if kv:
            k, v = kv.group(1), kv.group(2)
            keys[fq(section, k)] = v
            ordered.append((section, k, v))
    return keys, ordered


def fix_en_gb(dry_run: bool) -> int:
    """Rewrite American spellings in en-GB values to British. Returns # changed."""
    out_lines: list[str] = []
    section = ""
    changed = 0
    report: list[str] = []
    for raw in EN_GB.read_text(encoding="utf-8").splitlines():
        s = raw.strip()
        sec = SECTION_RE.match(s)
        if sec:
            section = sec.group(1)
            out_lines.append(raw)
            continue
        kv = KV_RE.match(s)
        if not kv:
            out_lines.append(raw)
            continue
        key, value = kv.group(1), kv.group(2)
        new_value, edits = us_to_uk_convert(value)
        if edits:
            changed += 1
            for src, dst in edits:
                report.append(f"  [en-GB] {fq(section, key)}: {src} -> {dst}")
            indent = raw[: len(raw) - len(raw.lstrip())]
            out_lines.append(f'{indent}{key} = "{new_value}"')
        else:
            out_lines.append(raw)
    if report:
        print(f"en-GB: {changed} value(s) Americanised -> British:")
        print("\n".join(report))
    else:
        print("en-GB: no American spellings found in values.")
    if not dry_run and changed:
        EN_GB.write_text("\n".join(out_lines) + "\n", encoding="utf-8")
    return changed


def parse_structured(path: Path) -> tuple[list[tuple[str, str]], list[str], dict[str, list[tuple[str, str]]]]:
    """Return (top_level_kvs, section_order, {section: kvs}) preserving file order."""
    top: list[tuple[str, str]] = []
    order: list[str] = []
    sections: dict[str, list[tuple[str, str]]] = {}
    section = ""
    for raw in path.read_text(encoding="utf-8").splitlines():
        s = raw.strip()
        if not s or s.startswith("#"):
            continue
        sec = SECTION_RE.match(s)
        if sec:
            section = sec.group(1)
            if section not in sections:
                order.append(section)
                sections[section] = []
            continue
        kv = KV_RE.match(s)
        if kv:
            (top if section == "" else sections[section]).append((kv.group(1), kv.group(2)))
    return top, order, sections


def _insert_ci(items: list, new, key_lower):
    """Insert `new` into `items` at the case-insensitive sorted position."""
    nk = key_lower(new)
    for i, existing in enumerate(items):
        if key_lower(existing) > nk:
            items.insert(i, new)
            return
    items.append(new)


def sync_en_us(dry_run: bool) -> int:
    """Regenerate en-US to mirror en-GB's structure, British->American.

    en-GB is the source of truth for which keys exist and in what order. For
    keys present in both, en-US keeps its own (US) wording; en-GB-only keys are
    added in their en-GB position; en-US-only keys/sections are slotted into the
    correct case-insensitive sorted position so the file stays ordered. Every
    value is run through the British->American converter.
    """
    us_keys, _ = parse_keys(EN_US)
    gb_keys, _ = parse_keys(EN_GB)
    gb_top, gb_order, gb_sections = parse_structured(EN_GB)
    us_top, us_order, us_sections = parse_structured(EN_US)

    added = [k for k in gb_keys if k not in us_keys]
    us_only = [k for k in us_keys if k not in gb_keys]

    def pick(section: str, key: str, gb_value: str) -> str:
        """Prefer en-US's own wording for shared keys; convert UK->US either way."""
        value = us_keys.get(fq(section, key), gb_value)
        return uk_to_us_convert(value)[0]

    # --- top-level keys: mirror en-GB, slot en-US-only keys in sorted order ---
    out_top: list[tuple[str, str]] = [(k, pick("", k, v)) for k, v in gb_top]
    gb_top_keys = {k for k, _ in gb_top}
    for k, v in us_top:
        if k not in gb_top_keys:
            _insert_ci(out_top, (k, uk_to_us_convert(v)[0]), lambda kv: kv[0].lower())

    # --- sections: mirror en-GB order/keys, merge en-US-only keys & sections ---
    out_sections: list[tuple[str, list[tuple[str, str]]]] = []
    for name in gb_order:
        gb_kvs = gb_sections[name]
        gb_section_keys = {k for k, _ in gb_kvs}
        merged = [(k, pick(name, k, v)) for k, v in gb_kvs]
        # en-US-only keys that belong to this (shared) section
        for k, v in us_sections.get(name, []):
            if k not in gb_section_keys:
                _insert_ci(merged, (k, uk_to_us_convert(v)[0]), lambda kv: kv[0].lower())
        out_sections.append((name, merged))

    # en-US-only sections (absent from en-GB): insert by ci header order
    gb_section_names = set(gb_order)
    for name in us_order:
        if name not in gb_section_names:
            kvs = [(k, uk_to_us_convert(v)[0]) for k, v in us_sections[name]]
            _insert_ci(out_sections, (name, kvs), lambda s: s[0].lower())

    # --- emit (top-level block, then one blank line before each section) ---
    lines: list[str] = [f'{k} = "{v}"' for k, v in out_top]
    for name, kvs in out_sections:
        lines.append("")
        lines.append(f"[{name}]")
        lines.extend(f'{k} = "{v}"' for k, v in kvs)

    print(f"en-US: +{len(added)} key(s) from en-GB, "
          f"{len(us_only)} en-US-only key(s) preserved (British->American applied).")
    for k in added:
        print(f"  [en-US] + {k}")
    if not dry_run:
        EN_US.write_text("\n".join(lines) + "\n", encoding="utf-8")
    return len(added)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--dry-run", action="store_true", help="report changes without writing")
    args = ap.parse_args()

    if not EN_US.exists() or not EN_GB.exists():
        print(f"error: expected files under {LOCALES}", file=sys.stderr)
        return 1

    print(f"Spelling map: {len(uk_to_us)} British->American word forms.\n")
    added = sync_en_us(args.dry_run)
    print()
    fixed = fix_en_gb(args.dry_run)
    print()
    mode = "DRY RUN (no files written)" if args.dry_run else "written"
    print(f"Done [{mode}]: +{added} en-US key(s), {fixed} en-GB value(s) corrected.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())