#!/usr/bin/env python3
"""CPU-only, data-free DFlash CT W4/W8A16 conversion; never overwrite artifacts."""
import os
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import argparse
import hashlib
import importlib.metadata
import json
from pathlib import Path
import re
import struct
import time

from mlp_int8 import headers, sha256

SOURCE = Path("/home/lab/models/ornith-modular/dflash-bf16")
REVISION = "5fc3b3d474760f18c516db87d84c37edbfd3ede6"
IGNORE = ["re:.*fc$", "re:.*norm.*", "re:.*embed_tokens.*", "re:.*lm_head.*",
          "re:.*self_attn\.(?:q_proj|k_proj|v_proj|qkv_proj)$"]
ELIGIBLE = re.compile(r"layers\.\d+\.(?:self_attn\.o_proj|mlp\.(?:gate_proj|up_proj|down_proj))\.weight$")


def tensor_hash(path, entry):
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        offset = 8 + struct.unpack("<Q", stream.read(8))[0]
        low, high = entry["data_offsets"]
        stream.seek(offset + low)
        remaining = high - low
        while remaining:
            block = stream.read(min(8 * 1024 * 1024, remaining))
            assert block
            digest.update(block)
            remaining -= len(block)
    return digest.hexdigest()


def collect(directory):
    result, locations = {}, {}
    for file in sorted(directory.glob("*.safetensors")):
        for name, entry in headers(file).items():
            assert name not in result, name
            result[name], locations[name] = entry, file
    return result, locations


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--bits", type=int, choices=(4, 8), required=True)
    args = parser.parse_args()
    output = SOURCE.parent / f"dflash-mlp-int{args.bits}"
    import torch
    from compressed_tensors.quantization import preset_name_to_scheme
    from compressed_tensors.utils import match_quantizable_tensors
    from llmcompressor import model_free_ptq

    assert torch.version.cuda is None and not torch.cuda.is_available()
    assert SOURCE.is_dir() and not output.exists(), "Refusing existing output"
    revision_file = SOURCE / ".cache/huggingface/download/model.safetensors.metadata"
    assert revision_file.read_text().splitlines()[0] == REVISION
    torch.set_num_threads(4)
    scheme = preset_name_to_scheme(f"W{args.bits}A16", ["Linear"])
    scheme.format = "pack-quantized"
    assert scheme.weights.num_bits == args.bits and scheme.weights.symmetric
    assert scheme.weights.group_size == 128
    assert scheme.input_activations is None and scheme.output_activations is None
    original, original_paths = collect(SOURCE)
    selected = {name for _, name in match_quantizable_tensors(original, IGNORE, scheme.targets)}
    expected_selected = {name for name in original if ELIGIBLE.fullmatch(name)}
    preserved = set(original) - selected
    assert selected == expected_selected and len(selected) == 24
    assert len(preserved) == 45 and "fc.weight" in preserved
    assert original["fc.weight"]["dtype"] == "BF16"
    for name in selected:
        shape = original[name]["shape"]
        assert len(shape) == 2 and shape[1] % 128 == 0 and shape[0] % 64 == 0
    source_config = json.loads((SOURCE / "config.json").read_text())
    assert source_config["architectures"] == ["DFlashDraftModel"]
    assert "quantization_config" not in source_config
    print(f"CPU-only W{args.bits}A16: 24 MLP/o_proj weights; 45 preserved including QKV and fc", flush=True)
    started = time.time()
    model_free_ptq(model_stub=str(SOURCE), save_directory=str(output), scheme=scheme,
                   ignore=IGNORE, max_workers=1, device="cpu")
    result, result_paths = collect(output)
    expected = set(preserved)
    for name in selected:
        prefix = name.removesuffix(".weight")
        expected.update(prefix + suffix for suffix in (".weight_packed", ".weight_scale", ".weight_shape"))
        rows, cols = original[name]["shape"]
        packed = result[prefix + ".weight_packed"]
        assert packed["dtype"] == "I32" and packed["shape"] == [rows, cols // (32 // args.bits)]
        assert result[prefix + ".weight_scale"]["shape"] == [rows, cols // 128]
    assert set(result) == expected, set(result) ^ expected
    preserved_hashes = {}
    for name in sorted(preserved):
        before, after = original[name], result[name]
        assert before["dtype"] == after["dtype"] and before["shape"] == after["shape"], name
        digest = tensor_hash(original_paths[name], before)
        assert digest == tensor_hash(result_paths[name], after), name
        preserved_hashes[name] = digest
    config = json.loads((output / "config.json").read_text())
    quant = config.pop("quantization_config")
    assert config == source_config, "Only quantization_config may change"
    assert quant["quant_method"] == "compressed-tensors"
    assert quant["format"] == "pack-quantized" and quant["quantization_status"] == "compressed"
    assert quant["ignore"] == IGNORE
    for group in quant["config_groups"].values():
        assert group["weights"]["num_bits"] == args.bits and group["weights"]["symmetric"]
        assert group["weights"]["group_size"] == 128
    def files_manifest(files):
        return {p.name: {"size": p.stat().st_size, "sha256": sha256(p)} for p in files}
    print("Structure and all 45 preserved payloads verified; hashing source/output", flush=True)
    source_manifest = files_manifest(sorted(p for p in SOURCE.iterdir() if p.is_file()))
    assert source_manifest["model.safetensors"]["sha256"] == revision_file.read_text().splitlines()[1]
    manifest = {
        "source": str(SOURCE), "source_upstream_repository": "z-lab/Qwen3.5-9B-DFlash",
        "source_upstream_revision": REVISION, "output": str(output), "device": "cpu",
        "algorithm": "RTN/memoryless_minmax", "scheme": f"Mixed W{args.bits}A16 symmetric group128 MLP/o_proj only; QKV and fc BF16",
        "quantization_config": quant, "linear_tensors_quantized": len(selected),
        "preserved_tensors_byte_identical": len(preserved), "quantized_tensor_names": sorted(selected),
        "preserved_tensor_sha256": preserved_hashes,
        "tokenizer": "Not supplied by original auxiliary checkpoint; use unchanged target tokenizer",
        "runtime_loader_validated": False, "quality_validated": False,
        "versions": {name: importlib.metadata.version(name) for name in ("torch", "llmcompressor", "compressed-tensors", "transformers", "safetensors")},
        "elapsed_seconds": time.time() - started, "source_files": source_manifest,
        "output_files": files_manifest(sorted(p for p in output.iterdir() if p.is_file())),
        "scripts_sha256": {name: sha256(Path(__file__).parent / name) for name in ("dflash_ptq.py", "mlp_int8.py")},
    }
    (output / "ptq-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    print(json.dumps({"output": str(output), "status": "artifact-verified-not-runtime-tested", "bits": args.bits,
                      "weight_bytes": sum(p.stat().st_size for p in output.glob("*.safetensors")),
                      "elapsed_seconds": manifest["elapsed_seconds"]}), flush=True)


if __name__ == "__main__":
    main()
