#!/usr/bin/env python3
"""Reproducible streaming HTTP benchmark; never changes engine configuration."""
import argparse, concurrent.futures, hashlib, importlib.metadata, json, os, platform, random, statistics, subprocess, sys, threading, time, urllib.request
from pathlib import Path

def http(url, payload=None, timeout=30):
    req=urllib.request.Request(url, data=None if payload is None else json.dumps(payload).encode(), headers={'Content-Type':'application/json'})
    with urllib.request.urlopen(req, timeout=timeout) as r: return json.load(r)

def stream(base, payload, timeout):
    started=time.perf_counter(); first=None; last=None; chunks=0; content=''; reasoning=''; usage={}; finish=None; done=False
    payload={**payload,'stream':True,'stream_options':{'include_usage':True}}
    req=urllib.request.Request(base+'/chat/completions',data=json.dumps(payload).encode(),headers={'Content-Type':'application/json'})
    try:
        with urllib.request.urlopen(req,timeout=timeout) as response:
            for line in response:
                if not line.startswith(b'data:'): continue
                value=line[5:].strip()
                if value==b'[DONE]':
                    done=True
                    break
                event=json.loads(value)
                if event.get('error'):
                    raise RuntimeError('SSE backend error: '+str(event['error']))
                if event.get('usage'):usage=event['usage']
                for choice in event.get('choices',[]):
                    delta=choice.get('delta',{}); text=delta.get('content') or ''; thought=delta.get('reasoning_content') or delta.get('reasoning') or ''
                    if text or thought:
                        now=time.perf_counter(); first=now if first is None else first; last=now;chunks+=1
                    content+=text;reasoning+=thought
                    if choice.get('finish_reason'):finish=choice['finish_reason']
        finished=time.perf_counter(); elapsed=finished-started; tokens=usage.get('completion_tokens')
        duration=(last-first) if first is not None and last is not None else None
        problems=[]
        if not done:problems.append('missing SSE [DONE]')
        if not finish:problems.append('missing finish_reason')
        if not isinstance(tokens,int) or tokens<=0:problems.append('missing positive completion token usage')
        result={'ok':not problems,'seconds':elapsed,'ttft_seconds':None if first is None else first-started,
                'started_monotonic':started,'first_token_monotonic':first,'finished_monotonic':finished,
                'decode_seconds':duration,'completion_tokens':tokens,'usage':usage,'finish_reason':finish,
                'decode_tok_s':((tokens-1)/duration if tokens and tokens>1 and duration and chunks==tokens else None),
                'decode_tok_s_estimate':((tokens-1)/duration if tokens and tokens>1 and duration else None),
                'e2e_tok_s':tokens/elapsed if tokens else None,'stream_chunks':chunks,
                'content':content,'reasoning_sha256':hashlib.sha256(reasoning.encode()).hexdigest(),
                'sse_done':done}
        if problems:result['error']='; '.join(problems)
        return result
    except Exception as e:
        return {'ok':False,'error':str(e),'seconds':time.perf_counter()-started}

def make_prompt(tok, total, output, seed, task='retrieval'):
    rng=random.Random(seed)
    codes=[f'K{rng.randrange(10000000,99999999)}' for _ in range(3)]
    prefix=f'Documento de prueba independiente {seed}. Recupera tres claves del documento.\n'
    suffix=('\nDevuelve SOLO un objeto JSON con las claves alpha, beta y gamma, sin explicacion.' if task=='retrieval' else
            '\nIgnora las claves. Escribe una lista numerada de 400 consejos detallados para escribir pruebas unitarias en Python. No resumas: desarrolla cada punto con una frase completa.')
    markers=[f'\nLa clave {k} es {v}.\n' for k,v in zip(['alpha','beta','gamma'],codes)]
    def tokens(text):
        rendered=tok.apply_chat_template([{'role':'user','content':text}],tokenize=False,add_generation_prompt=True,enable_thinking=False)
        return len(tok.encode(rendered,add_special_tokens=False))
    overhead=tokens(prefix+''.join(markers)+suffix)
    budget=max(1,total-output-overhead-16)
    filler='\n'.join(f'Registro {seed}-{i}: lote {rng.randrange(10000,99999)} archivado sin incidencias.' for i in range(budget//8+100))
    ids=tok.encode(filler,add_special_tokens=False)[:budget]
    cuts=[0,int(len(ids)*.05),int(len(ids)*.50),int(len(ids)*.95),len(ids)]
    text=prefix
    for i in range(4):
        text+=tok.decode(ids[cuts[i]:cuts[i+1]])
        if i<3:text+=markers[i]
    text+=suffix
    count=tokens(text)
    assert count+output<=total,(count,output,total)
    return text,dict(zip(['alpha','beta','gamma'],codes)),count

def retrieval_ok(content, expected):
    try:
        text=content.strip(); text=text[text.index('{'):text.rindex('}')+1]
        return json.loads(text)==expected
    except Exception:return False

def foreign_markers(content, other_expected):
    return sorted({value for expected in other_expected for value in expected.values() if value in content})

def main():
    p=argparse.ArgumentParser();p.add_argument('--base',required=True);p.add_argument('--model',required=True);p.add_argument('--out',required=True)
    p.add_argument('--lengths',default='4096');p.add_argument('--concurrency',default='1,3');p.add_argument('--repeats',type=int,default=5)
    p.add_argument('--max-output',type=int,default=256);p.add_argument('--timeout',type=int,default=1800);p.add_argument('--tokenizer',default='/home/lab/models/Ornith-1.5-9B-original')
    p.add_argument('--gpu',default='');p.add_argument('--duration',type=int,default=0);p.add_argument('--seed',type=int,default=4400)
    p.add_argument('--task',choices=['retrieval','throughput'],default='retrieval')
    args=p.parse_args();os.environ['CUDA_VISIBLE_DEVICES']='';os.environ['TOKENIZERS_PARALLELISM']='false'
    from transformers import AutoTokenizer
    tok=AutoTokenizer.from_pretrained(args.tokenizer,local_files_only=True)
    out=Path(args.out)
    if out.exists() and any(out.iterdir()):
        raise RuntimeError('Refusing to append to or overwrite existing benchmark evidence: '+str(out))
    out.mkdir(parents=True,exist_ok=True)
    manifest={'started_utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),
              'arguments':vars(args),'python':sys.executable,'python_version':platform.python_version(),
              'script_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
              'packages':{name:importlib.metadata.version(name) for name in ('transformers','tokenizers')},
              'protocol':'SSE DONE + finish_reason + positive completion usage; no output-directory reuse'}
    (out/'run-manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
    warm=stream(args.base,{'model':args.model,'messages':[{'role':'user','content':'Responde solamente OK.'}],'max_tokens':32,'temperature':0,'chat_template_kwargs':{'enable_thinking':False}},args.timeout)
    (out/'warmup.json').write_text(json.dumps(warm,indent=2));assert warm['ok'],warm
    records=[];samples=[];stop=threading.Event()
    def monitor():
        while not stop.is_set():
            if args.gpu:
                try:
                    raw=subprocess.check_output(['nvidia-smi','-i',args.gpu,'--query-gpu=memory.used,utilization.gpu,temperature.gpu,power.draw','--format=csv,noheader,nounits'],text=True,timeout=5)
                    samples.append({'time':time.time(),'values':raw.strip()})
                except Exception:pass
            stop.wait(2)
    thread=threading.Thread(target=monitor,daemon=True);thread.start();started=time.monotonic();iteration=0;completed=False
    try:
        while True:
            for length in map(int,args.lengths.split(',')):
                for concurrency in map(int,args.concurrency.split(',')):
                    for rep in range(args.repeats):
                        seed=args.seed+iteration*100000+length*10+rep*100+concurrency*10000
                        prompts=[make_prompt(tok,length,args.max_output,seed+i,args.task) for i in range(concurrency)]
                        payloads=[{'model':args.model,'messages':[{'role':'user','content':text}],'max_tokens':args.max_output,'temperature':0,'chat_template_kwargs':{'enable_thinking':False}} for text,_,_ in prompts]
                        t=time.perf_counter()
                        with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
                            results=list(pool.map(lambda data:stream(args.base,data,args.timeout),payloads))
                        elapsed=time.perf_counter()-t
                        for i,result in enumerate(results):
                            foreign=foreign_markers(result.get('content',''),[p[1] for j,p in enumerate(prompts) if j!=i])
                            result.update({'retrieval_pass':retrieval_ok(result.get('content',''),prompts[i][1]) if args.task=='retrieval' else None,
                                           'output_present':bool(result.get('content','').strip()),
                                           'foreign_request_markers':foreign,
                                           'expected':prompts[i][1] if args.task=='retrieval' else None,
                                           'prompt_sha256':hashlib.sha256(prompts[i][0].encode()).hexdigest(),
                                           'estimated_prompt_tokens':prompts[i][2],'request':i})
                        row={'length_budget':length,'concurrency':concurrency,'repeat':rep,'iteration':iteration,'seed':seed,'wall_seconds':elapsed,
                             'aggregate_tok_s':sum(r.get('completion_tokens') or 0 for r in results)/elapsed,'results':results}
                        records.append(row)
                        with (out/'results.jsonl').open('a') as f:f.write(json.dumps(row)+'\n')
                        print(json.dumps({'length':length,'c':concurrency,'rep':rep,'seconds':round(elapsed,2),'retrieval_pass':[r['retrieval_pass'] for r in results],'ttft':[r.get('ttft_seconds') for r in results],'decode_est':[r.get('decode_tok_s_estimate') for r in results]}),flush=True)
                        if any(not r['ok'] for r in results):raise RuntimeError('HTTP failure: stop escalating; inspect results')
            iteration+=1
            if not args.duration or time.monotonic()-started>=args.duration:break
        completed=True
    finally:
        stop.set();thread.join(timeout=6)
        (out/'gpu.json').write_text(json.dumps(samples,indent=2))
        summary={'arguments':vars(args),'wall_seconds':time.monotonic()-started,'batches':len(records),'completed':completed,'all_pass':completed and bool(records) and all(r['ok'] and not r.get('foreign_request_markers') and (r['retrieval_pass'] if args.task=='retrieval' else r['output_present']) for b in records for r in b['results']),
                 'note':'decode estimate uses completion tokens and first/last SSE timestamps; chunking can batch tokens. All output includes reasoning tokens when supplied by server.'}
        (out/'summary.json').write_text(json.dumps(summary,indent=2));print(json.dumps(summary),flush=True)
if __name__=='__main__':main()
