#!/usr/bin/env python3
"""Small deterministic quality/regression suite; not a general intelligence eval."""
import argparse, ast, json, re, subprocess, sys, time, urllib.request
from pathlib import Path

def request(base,payload):
    req=urllib.request.Request(base+'/chat/completions',data=json.dumps(payload).encode(),headers={'Content-Type':'application/json'})
    with urllib.request.urlopen(req,timeout=240) as r:return json.load(r)

def code_check(content, name, tests):
    match=re.search(r'```(?:python)?\s*\n(.*?)```',content,re.S);code=match.group(1) if match else content.strip()
    tree=ast.parse(code)
    forbidden=(ast.Import,ast.ImportFrom,ast.Attribute,ast.ClassDef,ast.With,ast.AsyncWith,ast.Global,ast.Nonlocal)
    for node in ast.walk(tree):
        if isinstance(node,forbidden):raise ValueError('unsafe AST')
        if isinstance(node,ast.Name) and node.id.startswith('__'):raise ValueError('dunder forbidden')
        if isinstance(node,ast.Call) and (not isinstance(node.func,ast.Name) or node.func.id not in ['sum','len','abs','min','max','range',name]):raise ValueError('call forbidden')
    runner="import resource\nresource.setrlimit(resource.RLIMIT_CPU,(1,1))\nresource.setrlimit(resource.RLIMIT_AS,(268435456,268435456))\nns={'__builtins__':{k:__builtins__.__dict__[k] for k in ['sum','len','abs','min','max','range']}}\nexec("+repr(code)+",ns)\n"+"\n".join(f"assert ns[{name!r}]({','.join(repr(x) for x in args)}) == {expected!r}" for args,expected in tests)
    result=subprocess.run([sys.executable,'-I','-S','-c',runner],cwd='/tmp',capture_output=True,timeout=3)
    return result.returncode==0

def main():
    p=argparse.ArgumentParser();p.add_argument('--base',required=True);p.add_argument('--model',required=True);p.add_argument('--out',required=True);args=p.parse_args()
    cases=[('multiply','Calcula 17 por 23. Responde solo el numero.',lambda x:x.strip()=='391'),
           ('arithmetic','Calcula (123 + 77) / 5. Responde solo el numero.',lambda x:x.strip()=='40'),
           ('json','Devuelve SOLO JSON valido con exactamente {"ok":true,"n":3}.',lambda x:json.loads(x)=={'ok':True,'n':3}),
           ('extract','Registro: Ana tiene 7 piezas y Luis 11. Devuelve solo JSON con total y personas.',lambda x:json.loads(x)=={'total':18,'personas':2}),
           ('square','Solo codigo Python sin explicacion: define square(n) que devuelve el cuadrado.',lambda x:code_check(x,'square',[([3],9),([-4],16),([0],0)])),
           ('sum_list','Solo codigo Python: define total(xs) que suma una lista, incluyendo lista vacia.',lambda x:code_check(x,'total',[([[1,2,3]],6),([[]],0),([[-2,1]],-1)]))]
    rows=[]
    for name,prompt,validator in cases:
        started=time.monotonic();content=None
        try:
            response=request(args.base,{'model':args.model,'messages':[{'role':'user','content':prompt}],'temperature':0,'max_tokens':1024,'chat_template_kwargs':{'enable_thinking':False}})
            choice=response['choices'][0];content=choice['message'].get('content') or ''
            row={'case':name,'pass':bool(validator(content)),'content':content,'finish_reason':choice['finish_reason'],'usage':response.get('usage')}
        except Exception as e:row={'case':name,'pass':False,'error':str(e),'content':content}
        row['seconds']=time.monotonic()-started;rows.append(row);print(json.dumps(row,ensure_ascii=False),flush=True)
    try:
        tools=[{'type':'function','function':{'name':'add','description':'Suma dos enteros; usa esta herramienta para calcular.', 'parameters':{'type':'object','properties':{'a':{'type':'integer'},'b':{'type':'integer'}},'required':['a','b']}}}]
        response=request(args.base,{'model':args.model,'messages':[{'role':'user','content':'Usa la herramienta add para sumar 17 y 25. No hagas el calculo sin herramienta.'}],'tools':tools,'tool_choice':'auto','temperature':0,'max_tokens':1024,'chat_template_kwargs':{'enable_thinking':False}})
        calls=response['choices'][0]['message'].get('tool_calls') or []
        passed=bool(calls) and calls[0]['function']['name']=='add' and json.loads(calls[0]['function']['arguments'])=={'a':17,'b':25}
        rows.append({'case':'tool_call','pass':passed,'tool_calls':calls})
    except Exception as e:rows.append({'case':'tool_call','pass':False,'error':str(e)})
    out=Path(args.out);out.parent.mkdir(parents=True,exist_ok=True)
    report={'model':args.model,'base':args.base,'results':rows,'pass':all(r['pass'] for r in rows),'passed':sum(r['pass'] for r in rows),'total':len(rows)}
    out.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');print(json.dumps({k:v for k,v in report.items() if k!='results'}),flush=True)
    raise SystemExit(0 if report['pass'] else 1)
if __name__=='__main__':main()
