#!/usr/bin/env python3
"""Authorized full-linear INT8 CPU conversion. No service or GPU operations."""
import os
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import hashlib
import importlib.metadata
import json
from pathlib import Path
import time

from mlp_int8 import headers, sha256, SOURCE, REVISION
from full_int8_plan import IGNORE

OUTPUT = Path("/home/lab/models/ornith-modular/full-int8-text")


def main():
    import torch
    from compressed_tensors.quantization import preset_name_to_scheme
    from compressed_tensors.utils import match_quantizable_tensors
    from llmcompressor import model_free_ptq

    assert torch.version.cuda is None and not torch.cuda.is_available()
    assert SOURCE.is_dir() and not OUTPUT.exists(), "Refusing existing output"
    torch.set_num_threads(4)
    scheme = preset_name_to_scheme("W8A16", ["Linear", "re:.*lm_head$"])
    scheme.format = "pack-quantized"
    assert scheme.weights.num_bits == 8 and scheme.weights.symmetric
    assert scheme.weights.group_size == 128
    assert scheme.input_activations is None and scheme.output_activations is None
    source_files = sorted(SOURCE.glob("*.safetensors"))
    source_headers, source_map = {}, {}
    for file in source_files:
        for name, entry in headers(file).items():
            assert name not in source_headers, name
            source_headers[name] = entry
            source_map[name] = file
    selected = {name for _, name in match_quantizable_tensors(source_headers, IGNORE, scheme.targets)}
    preserved = set(source_headers) - selected
    assert len(selected) == 249 and len(preserved) == 193, (len(selected), len(preserved))
    assert "lm_head.weight" in selected
    for name in selected:
        assert not any(part in name for part in ("embed_tokens", "norm", "conv1d", "mtp", "visual")), name
        shape = source_headers[name]["shape"]
        assert len(shape) == 2 and shape[1] % 128 == 0, (name, shape)
    print("CPU-only: 249 linear matrices including lm_head; 193 tensors preserved", flush=True)
    started = time.time()
    model_free_ptq(model_stub=str(SOURCE), save_directory=str(OUTPUT), scheme=scheme,
                   ignore=IGNORE, max_workers=1, device="cpu")
    print("Packed checkpoint written; verifying structure and unchanged tensor payloads", flush=True)
    result_headers, result_map = {}, {}
    for file in sorted(OUTPUT.glob("*.safetensors")):
        for name, entry in headers(file).items():
            assert name not in result_headers, name
            result_headers[name] = entry
            result_map[name] = file
    expected = set(preserved)
    for name in selected:
        prefix = name.removesuffix(".weight")
        expected.update(prefix + suffix for suffix in (".weight_packed", ".weight_scale", ".weight_shape"))
        rows, cols = source_headers[name]["shape"]
        packed = result_headers[prefix + ".weight_packed"]
        assert packed["dtype"] == "I32" and packed["shape"] == [rows, cols // 4]
        assert result_headers[prefix + ".weight_scale"]["shape"] == [rows, cols // 128]
    assert set(result_headers) == expected, set(result_headers) ^ expected

    def tensor_hash(path, entry):
        import struct
        digest = hashlib.sha256()
        with path.open("rb") as stream:
            offset = 8 + struct.unpack("<Q", stream.read(8))[0]
            low, high = entry["data_offsets"]
            stream.seek(offset + low)
            remaining = high - low
            while remaining:
                block = stream.read(min(8 * 1024 * 1024, remaining))
                assert block
                digest.update(block)
                remaining -= len(block)
        return digest.hexdigest()

    for name in sorted(preserved):
        before, after = source_headers[name], result_headers[name]
        assert before["dtype"] == after["dtype"] and before["shape"] == after["shape"], name
        assert tensor_hash(source_map[name], before) == tensor_hash(result_map[name], after), name
    print("All 193 excluded tensors byte-identical; checking metadata and file hashes", flush=True)
    config = json.loads((OUTPUT / "config.json").read_text())
    quant = config["quantization_config"]
    assert quant["quant_method"] == "compressed-tensors"
    assert quant["format"] == "pack-quantized" and quant["quantization_status"] == "compressed"
    for group in quant["config_groups"].values():
        assert "re:.*lm_head$" in group["targets"]
        assert group["weights"]["num_bits"] == 8 and group["weights"]["symmetric"]
        assert group["weights"]["group_size"] == 128
    assert sha256(SOURCE / "tokenizer.json") == sha256(OUTPUT / "tokenizer.json")
    def files_manifest(files):
        result = {}
        for file in files:
            print(f"Hashing {file.parent.name}/{file.name}", flush=True)
            result[file.name] = {"size": file.stat().st_size, "sha256": sha256(file)}
        return result
    source_manifest = files_manifest(source_files)
    output_manifest = files_manifest(sorted(p for p in OUTPUT.iterdir() if p.is_file()))
    script_dir = Path(__file__).parent
    manifest = {
        "source": str(SOURCE), "source_upstream_revision": REVISION,
        "output": str(OUTPUT), "device": "cpu", "algorithm": "RTN/memoryless_minmax",
        "scheme": "W8A16 symmetric group128 all eligible linear matrices including lm_head",
        "quantization_config": quant, "ignore": IGNORE,
        "linear_tensors_quantized": len(selected), "preserved_tensors_byte_identical": len(preserved),
        "quantized_tensor_names": sorted(selected), "preserved_tensor_names": sorted(preserved),
        "runtime_loader_validated": False,
        "versions": {name: importlib.metadata.version(name) for name in ("torch", "llmcompressor", "compressed-tensors", "transformers", "safetensors")},
        "elapsed_seconds": time.time()-started, "source_files": source_manifest, "output_files": output_manifest,
        "scripts_sha256": {name: sha256(script_dir / name) for name in ("full_int8.py", "full_int8_plan.py", "mlp_int8.py")},
    }
    (OUTPUT / "ptq-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    print(json.dumps({"output": str(OUTPUT), "status": "artifact-verified-not-runtime-tested", "quantized": len(selected), "preserved": len(preserved), "elapsed_seconds": manifest["elapsed_seconds"]}), flush=True)


if __name__ == "__main__":
    main()
