build_patch.py

#!/usr/bin/env python3
"""
build_patch.py — the reproducible-patch engine for LuckyMasterEN.

Reads `patch/manifest.toml` (the single source of truth for every file we patch),
mirrors the owner's OWN `originals/installed/` tree into `out/patched/`, applies each
declared op, and writes `out/patched/PATCH-LOG.txt` — a full audit of everything
patched (and everything declared-but-deferred). The output tree is the basis for the
later stages (xdelta/IPS delta, English-installer re-wrap).

Hard rule: we never ship SYGNAS bytes. This runs only against the user's own
`originals/`; the manifest + this engine + the patch sources are the redistributable
part, never the originals or the build output (`out/` is gitignored).

Stdlib only (Python 3.11+ for tomllib). Run from anywhere:
  python tools/build_patch.py [--originals DIR] [--out DIR] [--manifest FILE]

Manifest op tables (applied in this fixed order; rename always last):
  [[xvi]]        glob + src        — repack a translated cp932 Ini into each .Xvi
  [[text_keys]]  file + keys{}     — replace INI `KEY=` values, preserve all else
  [[text_subst]] file + subs[]     — literal find->replace pairs within a text file
  [[text_file]]  file + src        — replace a whole file with a tracked EN version
  [[binpatch]]   file + strings[]  — replace whole NUL-terminated strings in place
                                     (wide=true UTF-16LE | encoding="cp932" SJIS | latin1)
  [[pe_res]]     file + strings{}  — translate PE-resource menus/dialogs (lang 1041) via lief
  [[pak]]        file + member[]   — rebuild a PACKDATA .pak (calc data.pak): each member is
                                     either `subs[]` (decode .nut, find->replace, re-compress)
                                     or `src` (replace member bytes, e.g. an EN button PNG)
  [[mink_info]]  glob + subs[]     — rewrite the per-character Title=/… in .mink `info` chunks
                                     (decode the cracked info codec, find->replace, re-compress)
  [[rename]]     frm + to          — rename a file within the patched tree
  [[rename_map]] dir+glob+subs{}   — bulk-rename matching files by a substring map + rewrite the
                                     same substrings in a `refs` text file (assets + links in step)
Every op may set `active = false` to record intent without applying (logged DEFERRED).
String fields are templated with {install_root_jp}/{install_root_en} from [meta].
"""
import os, sys, re, shutil, tomllib, argparse

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from sygnas_repack import repack_acz, repack_packdata, repack_mink   # xvi / pak / mink_info ops
from sygnas_unpack import parse_acz, parse_packdata, mink_chunks, mink_info_decompress

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OP_ORDER = ['xvi', 'text_keys', 'text_subst', 'text_file', 'binpatch', 'asmpoke', 'pe_res',
            'pak', 'img_text', 'mink_info', 'rename', 'rename_map']


class PatchError(Exception):
    pass


def tmpl(s, meta):
    """Expand {install_root_jp}/{install_root_en} (and any [meta] key) in a string."""
    if not isinstance(s, str):
        return s
    for k, v in meta.items():
        s = s.replace('{' + k + '}', str(v))
    return s


# ── op implementations: each returns a list of human-readable log lines ──────────
def op_xvi(e, ctx):
    import glob
    log = []
    pattern = os.path.join(ctx['out'], tmpl(e['glob'], ctx['meta']))
    hits = sorted(glob.glob(pattern))
    if not hits:
        raise PatchError(f"xvi glob matched nothing: {e['glob']}")
    for xvi in hits:
        stem = os.path.splitext(os.path.basename(xvi))[0]
        src = os.path.join(REPO, tmpl(e['src'], ctx['meta']).format(stem=stem))
        if not os.path.exists(src):
            log.append(f"    SKIP {os.path.relpath(xvi, ctx['out'])} (no src {os.path.relpath(src, REPO)})")
            continue
        orig = open(xvi, 'rb').read()
        new_ini = open(src, 'rb').read()              # translated Ini, already cp932
        out = repack_acz(orig, new_ini)
        open(xvi, 'wb').write(out)
        log.append(f"    xvi  {os.path.relpath(xvi, ctx['out'])}  <- {os.path.relpath(src, REPO)}  "
                   f"(Ini {len(new_ini)}b, {len(orig)}->{len(out)}b)")
    return log


def op_text_keys(e, ctx):
    path = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    enc = e.get('encoding', 'utf-8')
    text = open(path, 'rb').read().decode(enc)
    log = [f"    keys {e['file']} ({enc})"]
    for key, val in e['keys'].items():
        val = tmpl(val, ctx['meta'])
        pat = re.compile(rf'(?m)^({re.escape(key)}=)[^\r\n]*(\r?)$')
        text, n = pat.subn(lambda m: m.group(1) + val + m.group(2), text)
        if n == 0:
            raise PatchError(f"text_keys {e['file']}: key {key!r} not found")
        log.append(f"        {key} = {val}  (x{n})")
    open(path, 'wb').write(text.encode(enc))
    return log


def op_text_subst(e, ctx):
    path = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    enc = e.get('encoding', 'utf-8')
    text = open(path, 'rb').read().decode(enc)
    log = [f"    subst {e['file']} ({enc})"]
    total = 0
    for sub in e['subs']:
        frm, to = tmpl(sub['from'], ctx['meta']), tmpl(sub['to'], ctx['meta'])
        n = text.count(frm)
        if n == 0 and sub.get('require', True):
            raise PatchError(f"text_subst {e['file']}: {frm!r} not found")
        text = text.replace(frm, to)
        total += n
        log.append(f"        {frm!r} -> {to!r}  (x{n})")
    open(path, 'wb').write(text.encode(enc))
    return log


def op_text_file(e, ctx):
    """Replace a whole file with a tracked EN version. We author sources as UTF-8/LF;
    `encoding` transcodes (e.g. cp932 for XP Notepad) and `crlf` normalizes line ends."""
    dst = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    src = os.path.join(REPO, tmpl(e['src'], ctx['meta']))
    if not os.path.exists(src):
        raise PatchError(f"text_file src missing: {e['src']}")
    old = os.path.getsize(dst) if os.path.exists(dst) else 0
    enc, crlf = e.get('encoding'), e.get('crlf', False)
    if enc is None and not crlf:
        shutil.copyfile(src, dst)                                  # raw byte copy
    else:
        text = open(src, encoding='utf-8', newline='').read()
        if crlf:
            text = text.replace('\r\n', '\n').replace('\n', '\r\n')
        open(dst, 'wb').write(text.encode(enc or 'utf-8'))
    tail = f" [{enc or 'raw'}{', crlf' if crlf else ''}]"
    return [f"    file {e['file']}  <- {e['src']}  ({old} -> {os.path.getsize(dst)}b){tail}"]


def _binpatch_enc(e):
    """Resolve the (encoding, char-size) for a binpatch entry. `wide=true` = UTF-16LE
    (2-byte NUL); else an `encoding` (default latin1) with a 1-byte NUL. Use
    `encoding="cp932"` for hardcoded JP drawn via the ANSI (*A) APIs — `old` encodes as
    Shift-JIS to match the bytes in the image, `new` is ASCII (which cp932 passes through
    unchanged) and shorter, so it fits + NUL-pads."""
    if e.get('wide', False):
        return 'utf-16-le', 2
    return e.get('encoding', 'latin1'), 1


def op_binpatch(e, ctx):
    path = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    enc, cs = _binpatch_enc(e)
    data = bytearray(open(path, 'rb').read())
    log = [f"    bin  {e['file']} ({enc})"]
    for s in e['strings']:
        old, new = tmpl(s['old'], ctx['meta']), tmpl(s['new'], ctx['meta'])
        oe, ne = old.encode(enc), new.encode(enc)
        if len(ne) > len(oe):
            raise PatchError(f"binpatch {e['file']}: new {new!r} longer than old {old!r}")
        term = b'\x00' * cs
        needle = oe + term                          # match a COMPLETE NUL-terminated string
        n = data.count(needle)
        if n != 1:
            raise PatchError(f"binpatch {e['file']}: {old!r} matched {n} complete strings (want 1)")
        i = data.find(needle)
        data[i:i + len(oe)] = ne + b'\x00' * (len(oe) - len(ne))   # shrink in place; tail->NUL
        log.append(f"        {old!r} -> {new!r}  @0x{i:x}  (pad {len(oe) - len(ne)}b)")
    open(path, 'wb').write(bytes(data))
    return log


def op_asmpoke(e, ctx):
    """Same-size in-place byte pokes + wide-string writes into a verified-zero cave.
    For .text/.rdata surgery beyond binpatch (relocating + repointing string literals,
    bumping immediates). `va` = virtual address; file offset = va - imagebase (default
    0x400000, valid when each section's RawPtr == VirtAddr-imagebase). Every poke verifies
    `old` before writing `new` (must be equal length); every wstr asserts the destination
    is currently all-zero (a mapped, unreferenced cave). NOT lief: a raw surgical edit, so
    the PE loads byte-for-byte unchanged on XP. Does NOT recompute the PE checksum (cosmetic
    on XP — stale-checksum EXEs load fine; a later pe_res op on the same file refreshes it)."""
    path = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    base = e.get('imagebase', 0x400000)
    data = bytearray(open(path, 'rb').read())
    log = [f"    poke {e['file']}"]
    for w in e.get('wstr', []):
        off = w['va'] - base
        enc = (w['text'] + '\x00').encode('utf-16-le')
        cur = bytes(data[off:off + len(enc)])
        if any(cur):
            raise PatchError(f"asmpoke {e['file']}: cave @0x{w['va']:x} not zero ({cur.hex()})")
        data[off:off + len(enc)] = enc
        log.append(f"        wstr @0x{w['va']:x} = {w['text']!r} ({len(enc)}b)")
    for p in e.get('poke', []):
        off = p['va'] - base
        old = bytes.fromhex(p['old'].replace(' ', ''))
        new = bytes.fromhex(p['new'].replace(' ', ''))
        if len(old) != len(new):
            raise PatchError(f"asmpoke {e['file']} @0x{p['va']:x}: {p['old']}/{p['new']} length differ")
        cur = bytes(data[off:off + len(old)])
        if cur != old:
            raise PatchError(f"asmpoke {e['file']} @0x{p['va']:x}: expected {old.hex()} got {cur.hex()}")
        data[off:off + len(new)] = new
        log.append(f"        poke @0x{p['va']:x} {p['old']} -> {p['new']}")
    open(path, 'wb').write(bytes(data))
    return log


def op_pe_res(e, ctx):
    """Translate PE-resource strings (lang 1041 menus, later dialogs) via lief — these are
    Unicode resources, so no ASCII constraint. `strings` maps JP source -> EN."""
    import pe_res
    path = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    mapping = {tmpl(k, ctx['meta']): tmpl(v, ctx['meta']) for k, v in e['strings'].items()}
    layout = {dlg: {(int(k) if str(k).lstrip('-').isdigit() else k): v for k, v in ov.items()}
              for dlg, ov in e.get('layout', {}).items()}
    res = pe_res.patch(path, path, mapping, layout)
    geo = f", geom:{','.join(layout)}" if layout else ""
    log = [f"    pe-res {e['file']}  ({len(res['hits'])} strings translated{geo})"]
    for jp, en in res['hits']:
        log.append(f"        {jp!r} -> {en!r}")
    if res['remaining_jp']:
        log.append(f"        !! {len(res['remaining_jp'])} menu string(s) still JP "
                   f"(unmapped key?): " + ", ".join(repr(x) for x in res['remaining_jp'][:12]))
    return log


def op_pak(e, ctx):
    """Rebuild a PACKDATA `.pak` (the calc `data.pak`). Each `[[pak.member]]` either carries
    `subs` (decode the .nut via the cracked codec, apply cp932 find->replace like text_subst —
    EN must be pure ASCII since the engine draws it with DrawTextA — then re-compress) or `src`
    (replace the member's bytes wholesale, e.g. a translated button PNG). We operate only on the
    user's own data.pak; only the EN strings / generated art are redistributable."""
    path = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    orig = open(path, 'rb').read()
    members = {name: data for name, data, _ in parse_packdata(orig)}   # name -> decoded bytes
    overrides = {}
    log = [f"    pak  {e['file']}"]
    for m in e.get('member', []):
        name = m['name']
        if name not in members:
            raise PatchError(f"pak {e['file']}: member {name!r} not found")
        if 'subs' in m:
            enc = m.get('encoding', 'cp932')
            text = members[name].decode(enc)
            log.append(f"        member {name} ({enc}, {len(m['subs'])} subs)")
            for sub in m['subs']:
                frm, to = tmpl(sub['from'], ctx['meta']), tmpl(sub['to'], ctx['meta'])
                n = text.count(frm)
                if n == 0 and sub.get('require', True):
                    raise PatchError(f"pak {e['file']} [{name}]: {frm!r} not found")
                text = text.replace(frm, to)
                log.append(f"            {frm!r} -> {to!r}  (x{n})")
            new = text.encode(enc)
            bad = [s for s in re.findall(r'"((?:[^"\\]|\\.)*)"', text) if any(ord(c) > 0x7f for c in s)]
            if bad:
                raise PatchError(f"pak {e['file']} [{name}]: non-ASCII string literal(s) remain "
                                 f"(DrawTextA mojibake risk): {bad[:6]}")
            overrides[name] = new
        elif 'src' in m:
            src = os.path.join(REPO, tmpl(m['src'], ctx['meta']))
            if not os.path.exists(src):
                raise PatchError(f"pak {e['file']}: member src missing: {m['src']}")
            overrides[name] = open(src, 'rb').read()
            log.append(f"        member {name} <- {m['src']}  ({len(overrides[name])}b)")
        else:
            raise PatchError(f"pak {e['file']}: member {name!r} has neither subs nor src")
    if e.get('gen') == 'calc_png':                       # retext the baked-in button labels
        try:
            import calc_png
        except ImportError as ex:
            raise PatchError(f"pak {e['file']}: gen=calc_png needs pillow (flake-provided): {ex}")
        log.append("        gen  calc_png (baked button labels)")
        for base in calc_png.SPECS:
            for nm in calc_png.variants(base):
                if nm not in members or nm in overrides:
                    continue
                new_png, spec, font = calc_png.generate(members[nm], nm)
                overrides[nm] = new_png
                log.append(f"            {nm:28} -> {spec['text']!r}  ({font}, {len(new_png)}b)")
    new_pak = repack_packdata(orig, overrides)
    open(path, 'wb').write(new_pak)
    log.append(f"        rebuilt {len(orig)} -> {len(new_pak)}b  ({len(overrides)} member(s) replaced)")
    return log


def op_mink_info(e, ctx):
    """Rewrite the per-character `Title=`/… strings inside `.mink` `info` chunks — the text the
    MinkIt Settings list (control 0x3ec) + the Preview "Title:" actually show (proven per-file
    metadata via the cracked info codec, NOT the engine's (none)/(unk.) fallbacks). `glob` selects
    the files; each carries one character's `info`, so we apply whichever `subs` match (cp932
    find->replace on the decoded text), re-compress, and rebuild the container. EN must be pure
    ASCII (locale rule: app-read text). Every sub must land on ≥1 file; every matched file must
    change exactly once."""
    import glob as _glob
    enc = e.get('encoding', 'cp932')
    subs = [(tmpl(s['old'], ctx['meta']), tmpl(s['new'], ctx['meta'])) for s in e['subs']]
    files = sorted(_glob.glob(os.path.join(ctx['out'], tmpl(e['glob'], ctx['meta']))))
    log = [f"    mink_info {e['glob']}  ({len(files)} file(s), {len(subs)} sub(s))"]
    if not files:
        raise PatchError(f"mink_info {e['glob']}: no files matched")
    hits = {old: 0 for old, _ in subs}
    for path in files:
        orig = open(path, 'rb').read()
        off, size = mink_chunks(orig)['info']
        text = mink_info_decompress(orig[off:off+size]).decode(enc)
        changed = []
        for old, new in subs:
            c = text.count(old)
            if c:
                text = text.replace(old, new); hits[old] += c
                changed.append(f"{old!r}->{new!r}")
        if len(changed) != 1:
            raise PatchError(f"mink_info {os.path.basename(path)}: matched {len(changed)} sub(s) "
                             f"(want exactly 1): {changed}")
        new_info = text.encode(enc)
        bad = [chr(b) for b in new_info if b > 0x7f]
        if bad:
            raise PatchError(f"mink_info {os.path.basename(path)}: non-ASCII in new info text "
                             f"(app-read; locale rule): {bad[:8]}")
        new_mink = repack_mink(orig, new_info)
        open(path, 'wb').write(new_mink)
        log.append(f"        {os.path.basename(path):<22} {', '.join(changed)}  "
                   f"({len(orig)}->{len(new_mink)}b)")
    missing = [old for old, n in hits.items() if n == 0]
    if missing:
        raise PatchError(f"mink_info {e['glob']}: sub(s) never matched any file: {missing}")
    return log


def op_rename(e, ctx):
    frm = os.path.join(ctx['out'], tmpl(e['frm'], ctx['meta']))
    to = os.path.join(ctx['out'], tmpl(e['to'], ctx['meta']))
    if not os.path.exists(frm):
        raise PatchError(f"rename src missing: {e['frm']}")
    os.makedirs(os.path.dirname(to), exist_ok=True)
    os.rename(frm, to)
    return [f"    mv   {e['frm']}  ->  {e['to']}"]


def op_img_text(e, ctx):
    """Translate baked-in text on a LOOSE image file (not a .pak member) — for the wallpaper
    section-header bars. Styling/geometry live in calc_png.WP_HEADERS (keyed by basename): the JP
    run is tile-erased (texture preserved) and EN drawn over it, from the user's OWN image. Format
    preserved (JPEG re-encoded q92; GIF kept with 1-bit transparency)."""
    try:
        import calc_png
        from PIL import Image
    except ImportError as ex:
        raise PatchError(f"img_text {e['file']}: needs pillow (flake-provided): {ex}")
    path = os.path.join(ctx['out'], tmpl(e['file'], ctx['meta']))
    name = os.path.basename(e['file'])
    if name not in calc_png.WP_HEADERS:
        raise PatchError(f"img_text {e['file']}: no calc_png.WP_HEADERS spec for {name!r}")
    im = Image.open(path).convert('RGBA')
    spec = calc_png.apply_header(im, name)
    log = [f"    img_text {e['file']}  -> {spec['text']!r}"]
    ext = os.path.splitext(path)[1].lower()
    if ext in ('.jpg', '.jpeg'):
        im.convert('RGB').save(path, 'JPEG', quality=92)
    elif ext == '.gif':                                  # 1-bit alpha: reserve palette idx 255 = transparent
        p = im.convert('RGB').quantize(colors=255)
        p.paste(255, (0, 0), im.split()[3].point(lambda a: 255 if a < 128 else 0))
        p.save(path, 'GIF', transparency=255)
    else:
        im.save(path, 'PNG')
    return log


def op_rename_map(e, ctx):
    """Bulk-rename files in `dir` matching `glob` by applying a shared substring map to their
    NAMES, and rewrite the SAME substrings inside a `refs` text file (the page that links them) —
    so a set of renamed assets and their references move in lockstep from one map. Used for the
    wallpaper JPGs (らき☆マス_<artist>_WxH.jpg -> LuckyMas_<romaji>_WxH.jpg + the HTML <a>/<img>)."""
    import glob as _glob
    d = os.path.join(ctx['out'], tmpl(e['dir'], ctx['meta']))
    subs = {tmpl(k, ctx['meta']): tmpl(v, ctx['meta']) for k, v in e['subs'].items()}
    log = [f"    rename_map {e['dir']}  ({len(subs)} name subs)"]
    n = 0
    for path in sorted(_glob.glob(os.path.join(d, e['glob']))):
        base = os.path.basename(path)
        new = base
        for frm, to in subs.items():
            new = new.replace(frm, to)
        if new != base:
            os.rename(path, os.path.join(os.path.dirname(path), new))
            n += 1
    log.append(f"        renamed {n} file(s) matching {e['glob']}")
    if e.get('refs'):
        rp = os.path.join(ctx['out'], tmpl(e['refs'], ctx['meta']))
        enc = e.get('refs_encoding', 'utf-8')
        text = open(rp, 'rb').read().decode(enc)
        total = 0
        for frm, to in subs.items():
            total += text.count(frm)
            text = text.replace(frm, to)
        open(rp, 'wb').write(text.encode(enc))
        log.append(f"        rewrote {total} ref(s) in {e['refs']}")
    return log


OPS = {'xvi': op_xvi, 'text_keys': op_text_keys, 'text_subst': op_text_subst,
       'text_file': op_text_file, 'binpatch': op_binpatch, 'asmpoke': op_asmpoke, 'pe_res': op_pe_res,
       'pak': op_pak, 'img_text': op_img_text, 'mink_info': op_mink_info,
       'rename': op_rename, 'rename_map': op_rename_map}


def build(originals, out, manifest_path):
    with open(manifest_path, 'rb') as f:
        man = tomllib.load(f)
    meta = man.get('meta', {})

    inst = os.path.join(originals, 'installed')
    if not os.path.isdir(os.path.join(inst, 'app')):
        raise PatchError(f"originals not found: expected {inst}/app (the user's own copy). "
                         f"See originals/README.md.")

    # 1) mirror the user's own tree so unpatched files pass through untouched
    if os.path.exists(out):
        shutil.rmtree(out)
    shutil.copytree(inst, out)

    # 2) apply ops in fixed order; collect a full audit log
    audit = ["LuckyMasterEN — patch build log",
             f"manifest: {os.path.relpath(manifest_path, REPO)}",
             f"install root: {meta.get('install_root_jp','?')} -> {meta.get('install_root_en','?')}",
             ""]
    ctx = {'out': out, 'meta': meta}
    applied = deferred = 0
    for kind in OP_ORDER:
        entries = man.get(kind, [])
        if not entries:
            continue
        audit.append(f"[{kind}]")
        for e in entries:
            title = e.get('note', e.get('file', e.get('glob', e.get('frm', ''))))
            if not e.get('active', True):
                audit.append(f"  DEFERRED: {title}")
                if e.get('todo'):
                    audit.append(f"    TODO: {e['todo']}")
                deferred += 1
                continue
            audit.append(f"  {title}")
            audit += OPS[kind](e, ctx)
            applied += 1
        audit.append("")

    summary = f"applied {applied} op(s), {deferred} deferred"
    audit.append(summary)
    open(os.path.join(out, 'PATCH-LOG.txt'), 'w', encoding='utf-8').write('\n'.join(audit) + '\n')
    return applied, deferred, summary


def main(argv):
    ap = argparse.ArgumentParser(description="apply the reproducible patch to a copy of the originals")
    ap.add_argument('--originals', default=os.path.join(REPO, 'originals'))
    ap.add_argument('--out', default=os.path.join(REPO, 'out', 'patched'))
    ap.add_argument('--manifest', default=os.path.join(REPO, 'patch', 'manifest.toml'))
    a = ap.parse_args(argv)
    try:
        applied, deferred, summary = build(a.originals, a.out, a.manifest)
    except PatchError as ex:
        print(f"build_patch: ERROR: {ex}", file=sys.stderr)
        return 1
    print(f"build_patch: {summary} -> {os.path.relpath(a.out, REPO)}/  (see PATCH-LOG.txt)")
    return 0


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))