#!/usr/bin/env python3
"""Create and verify a new CPU-only, MLP-only W8A16 artifact; never edit source."""
import os
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import hashlib
import importlib.metadata
import json
from pathlib import Path
import re
import struct
import sys
import time

SOURCE = Path("/home/lab/models/ornith-modular/original-text")
OUTPUT = Path("/home/lab/models/ornith-modular/mlp-int8-text")
REVISION = "489cb97981b8654bcfcf30ce1f94ed1b62e07b53"
IGNORE = [
    "re:.*lm_head.*", "re:.*embed_tokens.*", "re:.*norm.*",
    "re:.*visual.*", "re:.*conv1d.*", "re:.*linear_attn.*",
    "re:.*self_attn.*", "re:.*mtp.*",
]
MLP = re.compile(r"model\.language_model\.layers\.\d+\.mlp\.(gate_proj|up_proj|down_proj)\.weight$")


def headers(path):
    with path.open("rb") as stream:
        size = struct.unpack("<Q", stream.read(8))[0]
        value = json.loads(stream.read(size))
    return {k: v for k, v in value.items() if k != "__metadata__"}


def sha256(path):
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for block in iter(lambda: stream.read(8 * 1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def main():
    import torch
    from compressed_tensors.quantization import preset_name_to_scheme
    from llmcompressor import model_free_ptq

    assert torch.version.cuda is None, "Use the isolated CPU torch wheel"
    assert not torch.cuda.is_available()
    assert SOURCE.is_dir() and not OUTPUT.exists(), "Refusing existing destination"
    scheme = preset_name_to_scheme("W8A16", ["Linear"])
    assert scheme.weights.num_bits == 8 and scheme.weights.symmetric
    assert scheme.weights.group_size == 128
    assert scheme.input_activations is None and scheme.output_activations is None
    # Make the real on-disk pack format explicit, not merely a config claim.
    scheme.format = "pack-quantized"
    source_files = sorted(SOURCE.glob("*.safetensors"))
    source_headers = {}
    source_map = {}
    for file in source_files:
        for name, value in headers(file).items():
            assert name not in source_headers, f"duplicate tensor {name}"
            source_headers[name] = value
            source_map[name] = file
    mlp = {name for name in source_headers if MLP.fullmatch(name)}
    assert len(mlp) == 96, f"Expected 32 x 3 MLP weights, got {len(mlp)}"
    for name in mlp:
        shape = source_headers[name]["shape"]
        assert len(shape) == 2 and shape[1] % 128 == 0
    # Validate the tool's name matcher selects exactly the intended tensors.
    from compressed_tensors.utils import match_quantizable_tensors
    selected = {name for _, name in match_quantizable_tensors(source_headers, IGNORE, scheme.targets)}
    assert selected == mlp, f"Selection mismatch: extra={selected-mlp}, missing={mlp-selected}"
    print(f"CPU-only conversion: {len(mlp)} MLP tensors, {len(source_headers)-len(mlp)} preserved tensors", flush=True)
    started = time.time()
    model_free_ptq(model_stub=str(SOURCE), save_directory=str(OUTPUT),
                   scheme=scheme, ignore=IGNORE, max_workers=1, device="cpu")
    result_headers = {}
    result_map = {}
    for file in sorted(OUTPUT.glob("*.safetensors")):
        for name, value in headers(file).items():
            assert name not in result_headers, f"duplicate output tensor {name}"
            result_headers[name] = value
            result_map[name] = file
    expected = set(source_headers) - mlp
    for name in mlp:
        prefix = name.removesuffix(".weight")
        expected.update(prefix + suffix for suffix in (".weight_packed", ".weight_scale", ".weight_shape"))
    assert set(result_headers) == expected, f"Unexpected output tensor names: {set(result_headers)^expected}"
    # Verify all excluded tensor payloads are byte-identical, streaming raw bytes.
    def tensor_hash(path, entry):
        digest = hashlib.sha256()
        with path.open("rb") as stream:
            start = 8 + struct.unpack("<Q", stream.read(8))[0]
            low, high = entry["data_offsets"]
            stream.seek(start + low)
            remaining = high - low
            while remaining:
                block = stream.read(min(8 * 1024 * 1024, remaining))
                assert block
                digest.update(block)
                remaining -= len(block)
        return digest.hexdigest()
    for name in set(source_headers) - mlp:
        before, after = source_headers[name], result_headers[name]
        assert before["dtype"] == after["dtype"] and before["shape"] == after["shape"]
        assert tensor_hash(source_map[name], before) == tensor_hash(result_map[name], after), name
    for name in mlp:
        prefix = name.removesuffix(".weight")
        packed = result_headers[prefix + ".weight_packed"]
        out_features, in_features = source_headers[name]["shape"]
        assert packed["dtype"] == "I32" and packed["shape"] == [out_features, in_features // 4]
        assert result_headers[prefix + ".weight_scale"]["shape"] == [out_features, in_features // 128]
    config = json.loads((OUTPUT / "config.json").read_text())
    quant = config["quantization_config"]
    assert quant["quant_method"] == "compressed-tensors"
    assert quant["format"] == "pack-quantized" and quant["quantization_status"] == "compressed"
    assert sha256(SOURCE / "tokenizer.json") == sha256(OUTPUT / "tokenizer.json")
    manifest = {
        "source": str(SOURCE), "source_upstream_revision": REVISION,
        "output": str(OUTPUT), "device": "cpu", "algorithm": "RTN/static minmax",
        "scheme": "W8A16 symmetric group128, MLP only", "ignore": IGNORE,
        "mlp_tensors_quantized": len(mlp), "preserved_tensors_byte_identical": len(source_headers)-len(mlp),
        "runtime_loader_validated": False,
        "versions": {name: importlib.metadata.version(name) for name in ("torch", "llmcompressor", "compressed-tensors", "transformers", "safetensors")},
        "elapsed_seconds": time.time()-started,
        "source_files": {p.name: {"size": p.stat().st_size, "sha256": sha256(p)} for p in source_files},
        "output_files": {p.name: {"size": p.stat().st_size, "sha256": sha256(p)} for p in sorted(OUTPUT.iterdir()) if p.is_file()},
        "script_sha256": sha256(Path(__file__)),
    }
    (OUTPUT / "ptq-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    print(json.dumps({"output": str(OUTPUT), "status": "artifact-verified-not-runtime-tested", "quantized": len(mlp), "preserved": len(source_headers)-len(mlp), "elapsed_seconds": manifest["elapsed_seconds"]}), flush=True)


if __name__ == "__main__":
    main()
