#!/usr/bin/env python3
"""Extract native MTP-only CPU checkpoint and make immutable mixed CT variants."""
import os
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import argparse
import importlib.metadata
import json
from pathlib import Path
import re
import shutil
import time

from mlp_int8 import sha256
from dflash_ptq import collect, tensor_hash

SOURCE = Path("/home/lab/models/ornith-modular/original-text")
COMPACT = SOURCE.parent / "mtp-bf16-compact"
REVISION = "489cb97981b8654bcfcf30ce1f94ed1b62e07b53"
EMBED_HEAD = {"model.language_model.embed_tokens.weight", "lm_head.weight"}
IGNORE = ["re:.*fc$", "re:.*norm.*", "re:.*embed_tokens.*", "re:.*lm_head.*"]
ELIGIBLE = re.compile(r"mtp\.layers\.0\.(?:self_attn\.(?:q_proj|k_proj|v_proj|o_proj)|mlp\.(?:gate_proj|up_proj|down_proj))\.weight$")


def files_manifest(files):
    return {p.name: {"bytes": p.stat().st_size, "sha256": sha256(p)} for p in sorted(files)}


def versions():
    return {name: importlib.metadata.version(name) for name in
            ("torch", "llmcompressor", "compressed-tensors", "transformers", "safetensors")}


def script_hashes():
    return {name: sha256(Path(__file__).parent / name) for name in
            ("mtp_ptq.py", "dflash_ptq.py", "mlp_int8.py")}


def extract():
    from safetensors import safe_open
    from safetensors.torch import save_file
    assert not COMPACT.exists(), "Refusing existing compact checkpoint"
    original, original_paths = collect(SOURCE)
    selected = {name for name in original if name.startswith("mtp.")} | EMBED_HEAD
    assert selected <= set(original) and len(selected) == 17
    assert len(selected - EMBED_HEAD) == 15
    assert all(original[name]["dtype"] == "BF16" for name in selected)
    config = json.loads((SOURCE / "config.json").read_text())
    assert "quantization_config" not in config
    assert config["architectures"] == ["Qwen3_5ForConditionalGeneration"]
    started = time.time()
    COMPACT.mkdir()
    # Source support files are copied unchanged; tensor/component manifests are
    # regenerated to avoid claiming the compact directory is the whole target.
    support = ("config.json", "tokenizer.json", "tokenizer_config.json", "vocab.json",
               "merges.txt", "chat_template.jinja", "preprocessor_config.json",
               "processor_config.json", "video_preprocessor_config.json", "README.md")
    for name in support:
        if (SOURCE / name).is_file():
            shutil.copy2(SOURCE / name, COMPACT / name)
    weight_map = {}
    for shard, names in (("model-00001-of-00002.safetensors", EMBED_HEAD),
                         ("model-00002-of-00002.safetensors", selected - EMBED_HEAD)):
        tensors = {}
        for path in sorted({original_paths[name] for name in names}):
            with safe_open(path, framework="pt", device="cpu") as reader:
                for name in sorted(names):
                    if original_paths[name] == path:
                        tensors[name] = reader.get_tensor(name)
        print(f"Writing new compact shard {shard} with {len(tensors)} original tensors", flush=True)
        save_file(tensors, COMPACT / shard, metadata={"format": "pt"})
        weight_map.update({name: shard for name in tensors})
        del tensors
    index = {"metadata": {"total_size": sum(original[n]["data_offsets"][1]-original[n]["data_offsets"][0] for n in selected)},
             "weight_map": weight_map}
    (COMPACT / "model.safetensors.index.json").write_text(json.dumps(index, indent=2) + "\n")
    result, result_paths = collect(COMPACT)
    assert set(result) == selected
    hashes = {}
    for name in sorted(selected):
        assert result[name]["dtype"] == original[name]["dtype"]
        assert result[name]["shape"] == original[name]["shape"]
        digest = tensor_hash(original_paths[name], original[name])
        assert tensor_hash(result_paths[name], result[name]) == digest
        hashes[name] = digest
    for name in support:
        if (SOURCE / name).is_file():
            assert sha256(SOURCE / name) == sha256(COMPACT / name)
    manifest = {"source": str(SOURCE), "source_upstream_revision": REVISION,
                "output": str(COMPACT), "device": "cpu", "type": "native-MTP-subset-not-standalone-target",
                "tensor_count": len(selected), "all_tensors_byte_identical": True,
                "tensor_sha256": hashes, "runtime_loader_validated": False,
                "versions": versions(), "scripts_sha256": script_hashes(),
                "source_files": files_manifest(set(original_paths[name] for name in selected)),
                "output_files": files_manifest(p for p in COMPACT.iterdir() if p.is_file()),
                "elapsed_seconds": time.time() - started}
    (COMPACT / "extraction-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    print(json.dumps({"output": str(COMPACT), "status": "extraction-verified", "tensors": 17,
                      "payload_bytes": index["metadata"]["total_size"]}), flush=True)


def quantize(bits):
    from compressed_tensors.quantization import preset_name_to_scheme
    from compressed_tensors.utils import match_quantizable_tensors
    from llmcompressor import model_free_ptq
    output = SOURCE.parent / f"mtp-linear-int{bits}"
    assert not output.exists(), "Refusing existing quantized artifact"
    extraction = json.loads((COMPACT / "extraction-manifest.json").read_text())
    assert extraction["all_tensors_byte_identical"] and extraction["tensor_count"] == 17
    original, original_paths = collect(COMPACT)
    assert len(original) == 17
    scheme = preset_name_to_scheme(f"W{bits}A16", ["Linear"])
    scheme.format = "pack-quantized"
    assert scheme.weights.symmetric and scheme.weights.group_size == 128
    assert scheme.weights.num_bits == bits and scheme.input_activations is None
    selected = {n for _, n in match_quantizable_tensors(original, IGNORE, scheme.targets)}
    assert selected == {n for n in original if ELIGIBLE.fullmatch(n)} and len(selected) == 7
    preserved = set(original) - selected
    assert len(preserved) == 10 and "mtp.fc.weight" in preserved
    for name in selected:
        rows, cols = original[name]["shape"]
        assert rows % 64 == 0 and cols % 128 == 0
    started = time.time()
    print(f"CPU-only MTP W{bits}A16, seven linears; fc/norm/embedding/head remain BF16", flush=True)
    model_free_ptq(model_stub=str(COMPACT), save_directory=str(output), scheme=scheme,
                   ignore=IGNORE, max_workers=1, device="cpu")
    result, result_paths = collect(output)
    expected = set(preserved)
    for name in selected:
        prefix = name.removesuffix(".weight")
        expected.update(prefix+s for s in (".weight_packed", ".weight_scale", ".weight_shape"))
        rows, cols = original[name]["shape"]
        assert result[prefix+".weight_packed"]["shape"] == [rows, cols // (32//bits)]
        assert result[prefix+".weight_packed"]["dtype"] == "I32"
        assert result[prefix+".weight_scale"]["shape"] == [rows, cols // 128]
    assert set(result) == expected
    preserved_hashes = {}
    for name in sorted(preserved):
        assert result[name]["dtype"] == original[name]["dtype"]
        assert result[name]["shape"] == original[name]["shape"]
        digest = tensor_hash(original_paths[name], original[name])
        assert digest == extraction["tensor_sha256"][name]
        assert tensor_hash(result_paths[name], result[name]) == digest
        preserved_hashes[name] = digest
    config = json.loads((output / "config.json").read_text())
    quant = config.pop("quantization_config")
    assert config == json.loads((SOURCE / "config.json").read_text())
    assert quant["quant_method"] == "compressed-tensors" and quant["format"] == "pack-quantized"
    assert quant["quantization_status"] == "compressed" and quant["ignore"] == IGNORE
    for group in quant["config_groups"].values():
        assert group["weights"]["num_bits"] == bits and group["weights"]["symmetric"]
        assert group["weights"]["group_size"] == 128
    for name in ("tokenizer.json", "tokenizer_config.json", "chat_template.jinja"):
        assert sha256(SOURCE / name) == sha256(output / name)
    print("Packed structure, all preserved payloads and configuration verified; hashing files", flush=True)
    manifest = {"source": str(COMPACT), "source_upstream_revision": REVISION, "output": str(output),
                "device": "cpu", "scheme": f"mixed W{bits}A16 symmetric group128, seven MTP linear matrices only",
                "algorithm": "RTN/memoryless_minmax", "quantization_config": quant,
                "quantized_tensor_names": sorted(selected), "preserved_tensor_sha256": preserved_hashes,
                "source_extraction_manifest_sha256": sha256(COMPACT / "extraction-manifest.json"),
                "source_files": files_manifest(COMPACT.glob("*.safetensors")),
                "output_files": files_manifest(p for p in output.iterdir() if p.is_file()),
                "versions": versions(), "scripts_sha256": script_hashes(),
                "runtime_loader_validated": False, "quality_validated": False,
                "elapsed_seconds": time.time() - started}
    (output / "ptq-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    print(json.dumps({"output": str(output), "status": "artifact-verified-not-runtime-tested",
                      "weight_bytes": sum(p.stat().st_size for p in output.glob("*.safetensors")),
                      "elapsed_seconds": manifest["elapsed_seconds"]}), flush=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("operation", choices=("extract", "int8", "int4"))
    args = parser.parse_args()
    import torch
    assert torch.version.cuda is None and not torch.cuda.is_available()
    torch.set_num_threads(4)
    extract() if args.operation == "extract" else quantize(int(args.operation[3:]))
