#!/usr/bin/env bash
set -Eeuo pipefail

# ============================================================================
# Ornith P3 — Reasoning-budget / coding-reasoning benchmark
#
# Goal:
#   Find the smallest reasoning budget that still closes coding tasks correctly,
#   without the old failure mode where the model consumes the whole output
#   budget in reasoning_content and never emits a final answer.
#
# Profiles (per-request llama.cpp `thinking_budget_tokens`):
#   LOW    = 256
#   MEDIUM = 1024
#   HIGH   = 2048
#
# 3 rounds/profile, same generated exam for the same round across all profiles.
# Each exam has 8 deterministic coding/reasoning questions with an exact JSON
# final answer, so scoring is automatic.
#
# Metrics:
#   - score / 8
#   - exact-pass (8/8)
#   - final JSON format
#   - finish_reason stop vs length
#   - reasoning tokens
#   - final-content tokens
#   - TTFT / total latency / gen t/s
#
# Baseline restored on success OR failure:
#   3 x 256K, Q4/Q4, FA ON, DFlash OFF, cache ON, hard output 4096.
# ============================================================================

ROOT="${HOME}/bench/ornith"
P3ROOT="${ROOT}/p3"

IMAGE="lomelidev/llama.cpp:cu130-nccl-sse-faall-10974"
MODEL_DIR="${HOME}/models/ornith15-9b"
MODEL_FILE="Ornith-1.5-9B-Q4_K_L.gguf"

PORT=8120
MODEL_ALIAS="ornith-local"

KV_K="q4_0"
KV_V="q4_0"

# Small context on purpose: P3 isolates reasoning, not long-context behavior.
P3_CTX=32768
OUTPUT_LIMIT=4096
BASELINE_OUTPUT=4096
CACHE_RAM=32768

ROUNDS=3

LOW_BUDGET=256
MEDIUM_BUDGET=1024
HIGH_BUDGET=2048

RUN=""
MON_PID=""
RESTORE_ON_EXIT=0

SELF="$(readlink -f "$0")"

mkdir -p "$P3ROOT"

die() {
    echo >&2
    echo "ERROR: $*" >&2
    exit 1
}

warn() {
    echo "WARNING: $*" >&2
}

utc_stamp() {
    date -u +%Y%m%dT%H%M%SZ
}

need_cmd() {
    command -v "$1" >/dev/null 2>&1 || die "Falta comando: $1"
}

preflight_tools() {
    local c
    for c in docker curl jq python3 nvidia-smi tmux ss sha256sum awk sed grep column; do
        need_cmd "$c"
    done

    [[ -f "$MODEL_DIR/$MODEL_FILE" ]] || die "No existe modelo: $MODEL_DIR/$MODEL_FILE"
    docker image inspect "$IMAGE" >/dev/null 2>&1 || die "No existe imagen Docker: $IMAGE"
    tmux has-session -t llama 2>/dev/null || die "No existe tmux session 'llama'"

    # We specifically need dynamic per-request reasoning budgets.
    if [[ -d /opt/llama.cpp ]]; then
        if ! grep -Rqs --include='*.cpp' --include='*.h' \
            'thinking_budget_tokens' \
            /opt/llama.cpp/tools/server \
            /opt/llama.cpp/common 2>/dev/null; then
            die "Tu fuente local de llama.cpp no contiene thinking_budget_tokens"
        fi
    else
        warn "/opt/llama.cpp no existe; no pude verificar thinking_budget_tokens en fuente local"
    fi

    return 0
}

# ----------------------------------------------------------------------------
# Clean possible contamination
# ----------------------------------------------------------------------------

clean_old_clients() {
    echo
    echo "== Limpiando clientes que puedan contaminar :8120 =="

    pkill -TERM -f 'opencode run.*local-ornith/ornith-local' 2>/dev/null || true
    pkill -TERM -f 'opencode\.exe serve --stdio --port 0' 2>/dev/null || true
    sleep 1
    pkill -KILL -f 'opencode run.*local-ornith/ornith-local' 2>/dev/null || true
    pkill -KILL -f 'opencode\.exe serve --stdio --port 0' 2>/dev/null || true

    mapfile -t PIDS < <(
        ss -tnp 2>/dev/null \
          | grep ':8120' \
          | grep -oE 'pid=[0-9]+' \
          | cut -d= -f2 \
          | sort -u \
          || true
    )

    if ((${#PIDS[@]})); then
        echo "Cortando clientes conectados a :8120: ${PIDS[*]}"
        kill -TERM "${PIDS[@]}" 2>/dev/null || true
        sleep 1

        local pid
        for pid in "${PIDS[@]}"; do
            kill -KILL "$pid" 2>/dev/null || true
        done
    fi

    if ss -tnp 2>/dev/null | grep -q ':8120'; then
        echo "Conexiones restantes:"
        ss -tnp 2>/dev/null | grep ':8120' || true
    else
        echo "Sin conexiones a :8120"
    fi

    return 0
}

# ----------------------------------------------------------------------------
# llama lifecycle
# ----------------------------------------------------------------------------

start_p3_server() {
    docker rm -f ornith >/dev/null 2>&1 || true

    docker run -d \
      --name ornith \
      --gpus "device=0" \
      --ipc=host \
      -p "${PORT}:8080" \
      -v "${MODEL_DIR}:/models:ro" \
      -e GGML_CUDA_GRAPH_OPT=1 \
      "$IMAGE" \
      -m "/models/${MODEL_FILE}" \
      -a "$MODEL_ALIAS" \
      -ngl 999 \
      -fa on \
      -ctk "$KV_K" \
      -ctv "$KV_V" \
      -c "$P3_CTX" \
      -np 1 \
      --no-kv-unified \
      --cont-batching \
      --no-mmproj \
      --n-predict "$OUTPUT_LIMIT" \
      -b 4096 \
      -ub 2048 \
      --cache-ram "$CACHE_RAM" \
      --cache-prompt \
      --ctx-checkpoints 4 \
      --jinja \
      --metrics \
      --host 0.0.0.0 \
      --port 8080 \
      --temp 0.0 \
      --top-p 1.0 \
      --top-k 0 \
      --min-p 0 \
      >/dev/null

    local ok=0
    local _
    for _ in $(seq 1 180); do
        if curl -fsS "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
            ok=1
            break
        fi

        if ! docker ps --format '{{.Names}}' | grep -qx ornith; then
            break
        fi

        sleep 1
    done

    if [[ "$ok" != "1" ]]; then
        docker logs ornith 2>&1 | tail -160 || true
        die "llama no arrancó para P3"
    fi

    local logs
    logs="$(docker logs ornith 2>&1)"

    grep -Fq 'n_slots = 1, n_ctx_slot = 32768' <<<"$logs" \
      || die "P3 no quedó 1×32768"

    if grep -q 'no FlashAttention vector kernel' <<<"$logs"; then
        die "FA fallback detectado"
    fi

    if grep -Eqi 'cudaMalloc failed|out of memory|capping' <<<"$logs"; then
        die "OOM/capping detectado"
    fi

    return 0
}

restore_baseline() {
    echo
    echo "== Restaurando baseline 3x256K / Q4Q4 / FA / cache ON / output=4096 =="

    docker rm -f ornith >/dev/null 2>&1 || true

    docker run -d \
      --name ornith \
      --gpus "device=0" \
      --ipc=host \
      -p "${PORT}:8080" \
      -v "${MODEL_DIR}:/models:ro" \
      -e GGML_CUDA_GRAPH_OPT=1 \
      "$IMAGE" \
      -m "/models/${MODEL_FILE}" \
      -a "$MODEL_ALIAS" \
      -ngl 999 \
      -fa on \
      -ctk "$KV_K" \
      -ctv "$KV_V" \
      -c 786432 \
      -np 3 \
      --no-kv-unified \
      --cont-batching \
      --no-mmproj \
      --n-predict "$BASELINE_OUTPUT" \
      -b 4096 \
      -ub 2048 \
      --cache-ram "$CACHE_RAM" \
      --cache-prompt \
      --ctx-checkpoints 4 \
      --jinja \
      --metrics \
      --host 0.0.0.0 \
      --port 8080 \
      --temp 0.6 \
      --top-p 0.95 \
      --top-k 20 \
      --min-p 0 \
      >/dev/null

    local ok=0
    local _
    for _ in $(seq 1 180); do
        if curl -fsS "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
            ok=1
            break
        fi
        sleep 1
    done

    [[ "$ok" == "1" ]] || die "No pude restaurar baseline"

    curl -s "http://127.0.0.1:${PORT}/slots" \
      | jq -r '.[] | "slot=\(.id) ctx=\(.n_ctx) busy=\(.is_processing) max=\(.params.max_tokens)"'

    return 0
}

# ----------------------------------------------------------------------------
# Exam generation + request/scoring helpers
# ----------------------------------------------------------------------------

write_helpers() {
    cat > "$RUN/build_exam.py" <<'PY'
#!/usr/bin/env python3
import json
import random
import sys
from pathlib import Path

round_no = int(sys.argv[1])
prompt_path = Path(sys.argv[2])
expected_path = Path(sys.argv[3])

rng = random.Random(2026091700 + round_no)

# Q1: deterministic TypeScript trace
seed = rng.randint(17, 41)
x = seed
for i in range(1, 8):
    if (x + i) % 3 == 0:
        x += i * 2
    elif x % 2 == 0:
        x //= 2
    else:
        x += i - 1
q1 = str(x)

# Q6: deterministic filter/map/reduce trace
xs = [rng.randint(1, 18) for _ in range(8)]
vals = [v for v in xs if v % 2 == 0]
mapped = [v + i for i, v in enumerate(vals)]
q6 = str(sum(mapped))

expected = {
    "q1": q1,
    "q2": "B",
    "q3": "C",
    "q4": "C",
    "q5": "A,E,B,C,D",
    "q6": q6,
    "q7": "B",
    "q8": "C",
}

prompt = f"""You are taking a coding-reasoning benchmark.

Work through every question carefully in your private reasoning.
Your FINAL answer must be ONLY one valid JSON object with exactly these keys:
q1,q2,q3,q4,q5,q6,q7,q8

Do not put markdown around the JSON.
Do not explain anything in the final answer.
Values must be strings.

Q1 — TypeScript trace

let x = {seed};
for (let i = 1; i <= 7; i++) {{
  if ((x + i) % 3 === 0) {{
    x += i * 2;
  }} else if (x % 2 === 0) {{
    x = Math.floor(x / 2);
  }} else {{
    x += i - 1;
  }}
}}
What is the final decimal value of x?

Q2 — Go cancellation / blocked worker

A worker contains:

for job := range jobs {{
    select {{
    case <-ctx.Done():
        return
    default:
    }}

    out <- job.Execute()
}}

Which patch best prevents the worker from being stuck forever on the result send
after cancellation, without relying on a tiny buffer?

A) close(out) immediately before every send.
B) r := job.Execute(); select {{ case out <- r: case <-ctx.Done(): return }}
C) Add another ctx.Done() check immediately before job.Execute(), then keep
   the blocking `out <- ...`.
D) Make `out` buffered with capacity 1 and change nothing else.

Return only the letter.

Q3 — PostgreSQL latest paid order per user

Need exactly one row for every user, including users with no paid orders.
For users with paid orders, return the latest order by created_at, breaking ties
with the greatest order id.

Which query shape is correct?

A) INNER JOIN orders and GROUP BY users.id.
B) LEFT JOIN all paid orders, ORDER BY orders.created_at DESC LIMIT 1.
C) FROM users u LEFT JOIN LATERAL (
       SELECT id
       FROM orders
       WHERE user_id = u.id AND status = 'paid'
       ORDER BY created_at DESC, id DESC
       LIMIT 1
   ) o ON true
D) LEFT JOIN orders with status='paid', then SELECT max(order_id).

Return only the letter.

Q4 — Sliding-window counter

A bucketed sliding-window counter stores a ring of time buckets and a running sum.
`Increment()` rotates/clears expired buckets, but `GetCount()` merely returns the
running sum. If traffic becomes idle for longer than the whole window, a later
GetCount() can still report old events.

What is the required fix?

A) Make every bucket atomic; no clock changes are needed.
B) Double the number of buckets.
C) GetCount() must advance/expire buckets using the current clock before
   returning the sum.
D) Reset the whole counter after every Increment().

Return only the letter.

Q5 — JavaScript event-loop order

console.log("A");
Promise.resolve().then(() => console.log("B"));
queueMicrotask(() => console.log("C"));
setTimeout(() => console.log("D"), 0);
console.log("E");

Give the output order as one comma-separated string with no spaces.

Q6 — TypeScript data transform

const xs = {json.dumps(xs)};
const y = xs
  .filter(v => v % 2 === 0)
  .map((v, i) => v + i)
  .reduce((a, b) => a + b, 0);

What is y as a decimal integer?

Q7 — Go lock ordering

Function F locks mutex A and then mutex B.
Function G locks mutex B and then mutex A.
Both can run concurrently.

Which fix directly removes the lock-order cycle?

A) Add a short sleep before locking B in G.
B) Enforce the same global order everywhere, for example always A then B.
C) Change both mutexes to RWMutex but keep the opposite acquisition order.
D) Increase GOMAXPROCS.

Return only the letter.

Q8 — Idempotent payment webhook

A webhook can be delivered repeatedly. A successful event must update account
balance exactly once. The database supports transactions and a UNIQUE event_id.

Which design is strongest?

A) SELECT first; if absent, process; INSERT the event after commit.
B) Keep an in-memory set of processed event IDs.
C) In one transaction, first claim event_id with a UNIQUE insert
   (for example INSERT ... ON CONFLICT DO NOTHING RETURNING ...); only the
   transaction that successfully claims it performs the balance mutation,
   then commits.
D) Retry the entire handler three times regardless of duplicate status.

Return only the letter.

FINAL FORMAT EXAMPLE (values here are illustrative only):
{{"q1":"0","q2":"A","q3":"A","q4":"A","q5":"X","q6":"0","q7":"A","q8":"A"}}
"""

prompt_path.write_text(prompt)
expected_path.write_text(
    json.dumps(
        {
            "round": round_no,
            "seed": seed,
            "xs": xs,
            "expected": expected,
        },
        indent=2,
    ) + "\n"
)

print(json.dumps({
    "round": round_no,
    "seed": seed,
    "xs": xs,
    "expected": expected,
}))
PY

    cat > "$RUN/chat_stream.py" <<'PY'
#!/usr/bin/env python3
import http.client
import json
import sys
import time
from pathlib import Path

prompt_path = Path(sys.argv[1])
sse_path = Path(sys.argv[2])
response_path = Path(sys.argv[3])
port = int(sys.argv[4])
output_limit = int(sys.argv[5])
thinking_budget = int(sys.argv[6])

payload = {
    "model": "ornith-local",
    "messages": [
        {"role": "user", "content": prompt_path.read_text()}
    ],
    "stream": True,
    "max_tokens": output_limit,
    "temperature": 0.0,
    "top_p": 1.0,
    "top_k": 0,
    "seed": 1,
    "reasoning_format": "auto",
    "thinking_budget_tokens": thinking_budget,
}

body = json.dumps(payload).encode()

conn = http.client.HTTPConnection("127.0.0.1", port, timeout=1200)

t0 = time.monotonic()
epoch0 = time.time()

conn.request(
    "POST",
    "/v1/chat/completions",
    body=body,
    headers={
        "Content-Type": "application/json",
        "Content-Length": str(len(body)),
        "Connection": "close",
    },
)

resp = conn.getresponse()

first_delta = None
content_parts = []
reasoning_parts = []
finish_reason = None
last_obj = None

with sse_path.open("wb") as raw:
    while True:
        line = resp.readline()
        if not line:
            break

        raw.write(line)
        raw.flush()

        stripped = line.strip()

        if not stripped.startswith(b"data:"):
            continue

        data = stripped[5:].strip()

        if not data or data == b"[DONE]":
            continue

        try:
            obj = json.loads(data)
        except Exception:
            continue

        last_obj = obj

        choices = obj.get("choices") or []
        if not choices:
            continue

        choice = choices[0]
        delta = choice.get("delta") or {}

        content = delta.get("content")
        reasoning = delta.get("reasoning_content")

        if content:
            if first_delta is None:
                first_delta = time.monotonic()
            content_parts.append(content)

        if reasoning:
            if first_delta is None:
                first_delta = time.monotonic()
            reasoning_parts.append(reasoning)

        if choice.get("finish_reason") is not None:
            finish_reason = choice.get("finish_reason")

t1 = time.monotonic()
epoch1 = time.time()

obj = {
    "http_status": resp.status,
    "start_epoch": epoch0,
    "end_epoch": epoch1,
    "ttft_ms": None if first_delta is None else (first_delta - t0) * 1000.0,
    "total_ms": (t1 - t0) * 1000.0,
    "finish_reason": finish_reason,
    "content": "".join(content_parts),
    "reasoning_content": "".join(reasoning_parts),
    "last_chunk": last_obj,
}

response_path.write_text(json.dumps(obj, indent=2) + "\n")
conn.close()

if resp.status < 200 or resp.status >= 300:
    sys.exit(2)
PY

    cat > "$RUN/poll_slot.py" <<'PY'
#!/usr/bin/env python3
import json
import os
import sys
import time
import urllib.request
from pathlib import Path

pid = int(sys.argv[1])
port = int(sys.argv[2])
jsonl_path = Path(sys.argv[3])
summary_path = Path(sys.argv[4])

summary = {
    "actual_slot": None,
    "task_id": None,
    "prompt_tokens": 0,
    "processed_tokens": 0,
    "cached_tokens": 0,
    "decoded_tokens": 0,
    "max_tokens": None,
}

def alive(p):
    try:
        os.kill(p, 0)
        return True
    except OSError:
        return False

def fetch_slots():
    try:
        with urllib.request.urlopen(
            f"http://127.0.0.1:{port}/slots",
            timeout=0.5,
        ) as r:
            return json.loads(r.read())
    except Exception:
        return []

seen_task = None
idle_after = 0

with jsonl_path.open("w") as out:
    while True:
        slots = fetch_slots()
        active = [s for s in slots if s.get("is_processing")]

        target = None

        if seen_task is not None:
            for s in active:
                if s.get("id_task") == seen_task:
                    target = s
                    break

        if target is None and active:
            target = active[0]
            seen_task = target.get("id_task")

        if target is not None:
            rec = {
                "ts": time.time(),
                "id": target.get("id"),
                "id_task": target.get("id_task"),
                "n_ctx": target.get("n_ctx"),
                "prompt": target.get("n_prompt_tokens") or 0,
                "processed": target.get("n_prompt_tokens_processed") or 0,
                "cache": target.get("n_prompt_tokens_cache") or 0,
                "decoded": ((target.get("next_token") or [{}])[0].get("n_decoded") or 0),
                "max_tokens": (target.get("params") or {}).get("max_tokens"),
            }

            out.write(json.dumps(rec) + "\n")
            out.flush()

            summary["actual_slot"] = rec["id"]
            summary["task_id"] = rec["id_task"]
            summary["prompt_tokens"] = max(summary["prompt_tokens"], rec["prompt"])
            summary["processed_tokens"] = max(summary["processed_tokens"], rec["processed"])
            summary["cached_tokens"] = max(summary["cached_tokens"], rec["cache"])
            summary["decoded_tokens"] = max(summary["decoded_tokens"], rec["decoded"])
            summary["max_tokens"] = rec["max_tokens"]
            idle_after = 0
        elif not alive(pid):
            idle_after += 1

        if not alive(pid) and idle_after >= 3:
            break

        time.sleep(0.02)

summary_path.write_text(json.dumps(summary, indent=2) + "\n")
PY

    cat > "$RUN/score.py" <<'PY'
#!/usr/bin/env python3
import http.client
import json
import sys
from pathlib import Path

response_path = Path(sys.argv[1])
expected_path = Path(sys.argv[2])
result_path = Path(sys.argv[3])
port = int(sys.argv[4])

response = json.loads(response_path.read_text())
expected_obj = json.loads(expected_path.read_text())
expected = expected_obj["expected"]

content = response.get("content") or ""
reasoning = response.get("reasoning_content") or ""

def parse_first_json_object(text):
    dec = json.JSONDecoder()
    for i, ch in enumerate(text):
        if ch != "{":
            continue
        try:
            obj, _ = dec.raw_decode(text[i:])
            if isinstance(obj, dict):
                return obj
        except Exception:
            pass
    return None

parsed = parse_first_json_object(content)

answers = {}
if isinstance(parsed, dict):
    for k in expected:
        if k in parsed:
            answers[k] = str(parsed[k]).strip()

per_question = {
    k: (answers.get(k) == str(v))
    for k, v in expected.items()
}

score = sum(per_question.values())
format_ok = (
    isinstance(parsed, dict)
    and set(parsed.keys()) == set(expected.keys())
)

def tokenize_count(text):
    if not text:
        return 0

    body = json.dumps({"content": text}).encode()
    conn = http.client.HTTPConnection("127.0.0.1", port, timeout=60)
    conn.request(
        "POST",
        "/tokenize",
        body=body,
        headers={
            "Content-Type": "application/json",
            "Content-Length": str(len(body)),
        },
    )
    r = conn.getresponse()
    raw = r.read()
    conn.close()

    if r.status < 200 or r.status >= 300:
        return None

    obj = json.loads(raw)
    tokens = obj.get("tokens")
    if isinstance(tokens, list):
        return len(tokens)
    if isinstance(tokens, int):
        return tokens
    return None

reasoning_tokens = tokenize_count(reasoning)
content_tokens = tokenize_count(content)

last = response.get("last_chunk") or {}
timings = last.get("timings") or {}
usage = last.get("usage") or {}

result = {
    "score": score,
    "max_score": len(expected),
    "pass": score == len(expected),
    "format_ok": format_ok,
    "closure_ok": (
        response.get("finish_reason") == "stop"
        and bool(content.strip())
    ),
    "finish_reason": response.get("finish_reason"),
    "answers": answers,
    "expected": expected,
    "per_question": per_question,
    "content": content,
    "reasoning_content": reasoning,
    "reasoning_tokens": reasoning_tokens,
    "content_tokens": content_tokens,
    "ttft_ms": response.get("ttft_ms"),
    "total_ms": response.get("total_ms"),
    "timings": timings,
    "usage": usage,
    "prompt_tps": timings.get("prompt_per_second"),
    "gen_tps": timings.get("predicted_per_second"),
}

result_path.write_text(json.dumps(result, indent=2) + "\n")
PY

    chmod +x \
      "$RUN/build_exam.py" \
      "$RUN/chat_stream.py" \
      "$RUN/poll_slot.py" \
      "$RUN/score.py"
}

# ----------------------------------------------------------------------------
# Monitor / tmux
# ----------------------------------------------------------------------------

start_monitor() {
    (
        echo "timestamp,mem_available_kib,load1,psi_cpu,psi_mem,psi_io,gpu_util,gpu_used_mib,gpu_free_mib,power_w,temp_c"

        while true; do
            local_ts="$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)"
            mem="$(awk '/MemAvailable:/ {print $2}' /proc/meminfo)"
            load1="$(awk '{print $1}' /proc/loadavg)"
            pcpu="$(awk '/some/ {for(i=1;i<=NF;i++) if($i ~ /^avg10=/){split($i,a,"=");print a[2]}}' /proc/pressure/cpu)"
            pmem="$(awk '/some/ {for(i=1;i<=NF;i++) if($i ~ /^avg10=/){split($i,a,"=");print a[2]}}' /proc/pressure/memory)"
            pio="$(awk '/some/ {for(i=1;i<=NF;i++) if($i ~ /^avg10=/){split($i,a,"=");print a[2]}}' /proc/pressure/io)"
            gpu="$(nvidia-smi -i 0 \
              --query-gpu=utilization.gpu,memory.used,memory.free,power.draw,temperature.gpu \
              --format=csv,noheader,nounits)"
            echo "${local_ts},${mem},${load1},${pcpu},${pmem},${pio},${gpu}"
            sleep 1
        done
    ) > "$RUN/system.csv" &

    MON_PID=$!
    echo "$MON_PID" > "$RUN/monitor.pid"
}

stop_monitor() {
    if [[ -n "${MON_PID:-}" ]]; then
        kill "$MON_PID" 2>/dev/null || true
        wait "$MON_PID" 2>/dev/null || true
        MON_PID=""
    fi
}

ensure_window() {
    local name="$1"
    if ! tmux list-windows -t llama -F '#{window_name}' | grep -qx "$name"; then
        tmux new-window -d -t llama -n "$name"
    fi
}

setup_tmux() {
    local w
    for w in Log Check Test Monitor OpenCode; do
        ensure_window "$w"
    done

    tmux send-keys -t llama:Log C-c
    tmux send-keys -t llama:Log \
      'while true; do clear; docker logs -f ornith 2>&1; sleep 1; done' C-m

    tmux send-keys -t llama:Check C-c
    tmux send-keys -t llama:Check \
      "watch -n 1 '$SELF dashboard'" C-m

    tmux send-keys -t llama:OpenCode C-c
    tmux send-keys -t llama:OpenCode \
      "watch -n 1 '$SELF slots'" C-m
}

# ----------------------------------------------------------------------------
# Progress / views
# ----------------------------------------------------------------------------

update_progress() {
    local profile="$1"
    local round="$2"
    local phase="$3"
    local budget="$4"

    python3 - "$RUN/progress.json" "$profile" "$round" "$phase" "$budget" <<'PY'
import json, os, sys, time

path, profile, rnd, phase, budget = sys.argv[1:]

obj = {
    "profile": profile,
    "round": int(rnd),
    "phase": phase,
    "thinking_budget_tokens": int(budget),
    "updated_epoch": time.time(),
}

tmp = path + ".tmp"
with open(tmp, "w") as f:
    json.dump(obj, f)
os.replace(tmp, path)
PY
}

dashboard() {
    [[ -f "$ROOT/LAST_P3" ]] || {
        echo "No hay LAST_P3 todavía."
        return
    }

    local d
    d="$(cat "$ROOT/LAST_P3")"

    clear
    echo "ORNITH P3 — REASONING BUDGET / CODING"
    echo "============================================================================"
    echo "$d"
    echo

    if [[ -f "$d/progress.json" ]]; then
        jq -r '
          "Profile : \(.profile)
Round   : \(.round)
Phase   : \(.phase)
Budget  : \(.thinking_budget_tokens) tokens"
        ' "$d/progress.json"
    else
        echo "Esperando..."
    fi

    echo
    echo "RESULTADOS"
    echo "============================================================================"
    if [[ -s "$d/summary.tsv" ]]; then
        tail -n 12 "$d/summary.tsv" | column -t -s $'\t'
    fi

    echo
    echo "GPU0"
    echo "============================================================================"
    nvidia-smi -i 0 \
      --query-gpu=utilization.gpu,memory.used,memory.free,power.draw,temperature.gpu \
      --format=csv,noheader
}

slots_view() {
    curl -s "http://127.0.0.1:${PORT}/slots" \
      | jq -r '
        .[] |
        "SLOT \(.id) task=\(.id_task) busy=\(.is_processing)
ctx=\(.n_ctx)
prompt=\(.n_prompt_tokens)
processed=\(.n_prompt_tokens_processed)
cache=\(.n_prompt_tokens_cache)
decoded=\(.next_token[0].n_decoded // 0)
max_tokens=\(.params.max_tokens)
---------------------------------------------"
      '
}

# ----------------------------------------------------------------------------
# One profile/round
# ----------------------------------------------------------------------------

run_case() {
    local profile="$1"
    local budget="$2"
    local round="$3"

    local case_dir="$RUN/cases/$profile/round-$(printf '%02d' "$round")"
    mkdir -p "$case_dir"

    update_progress "$profile" "$round" "build-exam" "$budget"

    python3 "$RUN/build_exam.py" \
      "$round" \
      "$case_dir/prompt.txt" \
      "$case_dir/expected.json" \
      > "$case_dir/exam.json"

    update_progress "$profile" "$round" "request" "$budget"

    local log_before
    log_before="$(docker logs ornith 2>&1 | wc -l)"

    python3 "$RUN/chat_stream.py" \
      "$case_dir/prompt.txt" \
      "$case_dir/response.sse" \
      "$case_dir/response.json" \
      "$PORT" \
      "$OUTPUT_LIMIT" \
      "$budget" &
    local req_pid=$!

    python3 "$RUN/poll_slot.py" \
      "$req_pid" \
      "$PORT" \
      "$case_dir/slots.jsonl" \
      "$case_dir/slots-summary.json" &
    local poll_pid=$!

    local rc=0
    wait "$req_pid" || rc=$?
    wait "$poll_pid" || true

    docker logs ornith 2>&1 \
      | tail -n "+$((log_before + 1))" \
      > "$case_dir/llama.log"

    if [[ "$rc" != "0" ]]; then
        cat "$case_dir/response.json" 2>/dev/null || true
        tail -120 "$case_dir/llama.log" || true
        die "$profile round=$round request rc=$rc"
    fi

    local capping fa_warn
    capping="$(grep -ci 'capping' "$case_dir/llama.log" || true)"
    fa_warn="$(grep -c 'no FlashAttention vector kernel' "$case_dir/llama.log" || true)"

    [[ "$capping" == "0" ]] || die "$profile round=$round tuvo capping"
    [[ "$fa_warn" == "0" ]] || die "$profile round=$round tuvo FA fallback"

    python3 "$RUN/score.py" \
      "$case_dir/response.json" \
      "$case_dir/expected.json" \
      "$case_dir/scored.json" \
      "$PORT"

    python3 - \
      "$profile" "$budget" "$round" \
      "$case_dir/scored.json" \
      "$case_dir/slots-summary.json" \
      "$RUN/results.jsonl" \
      "$RUN/summary.tsv" <<'PY'
import json
import sys

profile, budget, rnd, scored_path, slots_path, jsonl_path, tsv_path = sys.argv[1:]

s = json.load(open(scored_path))
slots = json.load(open(slots_path))

row = {
    "profile": profile,
    "budget": int(budget),
    "round": int(rnd),
    "score": s.get("score"),
    "max_score": s.get("max_score"),
    "pass": s.get("pass"),
    "format_ok": s.get("format_ok"),
    "closure_ok": s.get("closure_ok"),
    "finish_reason": s.get("finish_reason"),
    "reasoning_tokens": s.get("reasoning_tokens"),
    "content_tokens": s.get("content_tokens"),
    "decoded_tokens": slots.get("decoded_tokens"),
    "prompt_tokens": slots.get("prompt_tokens"),
    "ttft_ms": s.get("ttft_ms"),
    "total_ms": s.get("total_ms"),
    "prompt_tps": s.get("prompt_tps"),
    "gen_tps": s.get("gen_tps"),
    "per_question": s.get("per_question"),
}

with open(jsonl_path, "a") as f:
    f.write(json.dumps(row) + "\n")

wrong = ",".join(
    k for k, ok in (row["per_question"] or {}).items()
    if not ok
)

fields = [
    row["profile"],
    row["budget"],
    row["round"],
    f'{row["score"]}/{row["max_score"]}',
    "PASS" if row["pass"] else "FAIL",
    "yes" if row["format_ok"] else "no",
    "yes" if row["closure_ok"] else "no",
    row["finish_reason"] or "",
    row["reasoning_tokens"],
    row["content_tokens"],
    row["decoded_tokens"],
    f'{row["ttft_ms"]:.2f}' if isinstance(row["ttft_ms"], (int, float)) else "",
    f'{row["total_ms"]:.2f}' if isinstance(row["total_ms"], (int, float)) else "",
    f'{row["prompt_tps"]:.2f}' if isinstance(row["prompt_tps"], (int, float)) else "",
    f'{row["gen_tps"]:.2f}' if isinstance(row["gen_tps"], (int, float)) else "",
    wrong or "-",
]

with open(tsv_path, "a") as f:
    f.write("\t".join("" if x is None else str(x) for x in fields) + "\n")
PY

    echo
    echo "RESULT $profile budget=$budget round=$round"
    jq '{
      score,
      max_score,
      pass,
      format_ok,
      closure_ok,
      finish_reason,
      reasoning_tokens,
      content_tokens,
      ttft_ms,
      total_ms,
      prompt_tps,
      gen_tps,
      per_question,
      answers
    }' "$case_dir/scored.json"

    return 0
}

run_profile() {
    local profile="$1"
    local budget="$2"

    echo
    echo "============================================================================"
    echo "P3-$profile — thinking_budget_tokens=$budget"
    echo "============================================================================"

    # Fresh server per profile so no profile benefits from prior cache state.
    start_p3_server

    local r
    for r in $(seq 1 "$ROUNDS"); do
        run_case "$profile" "$budget" "$r"
    done

    return 0
}

# ----------------------------------------------------------------------------
# Report
# ----------------------------------------------------------------------------

make_report() {
    python3 - "$RUN" <<'PY'
from pathlib import Path
import json
import statistics
import sys

run = Path(sys.argv[1])

rows = [
    json.loads(x)
    for x in (run / "results.jsonl").read_text().splitlines()
    if x.strip()
]

order = ["LOW", "MEDIUM", "HIGH"]

def med(xs):
    xs = [float(x) for x in xs if isinstance(x, (int, float))]
    return statistics.median(xs) if xs else None

summary = {}

for profile in order:
    rs = [r for r in rows if r["profile"] == profile]
    if not rs:
        continue

    summary[profile] = {
        "budget": rs[0]["budget"],
        "samples": len(rs),
        "exact_passes": sum(1 for r in rs if r.get("pass")),
        "exact_pass_rate_pct": 100.0 * sum(1 for r in rs if r.get("pass")) / len(rs),
        "closure_passes": sum(1 for r in rs if r.get("closure_ok")),
        "closure_rate_pct": 100.0 * sum(1 for r in rs if r.get("closure_ok")) / len(rs),
        "format_passes": sum(1 for r in rs if r.get("format_ok")),
        "median_score": med([r.get("score") for r in rs]),
        "median_reasoning_tokens": med([r.get("reasoning_tokens") for r in rs]),
        "median_content_tokens": med([r.get("content_tokens") for r in rs]),
        "median_ttft_ms": med([r.get("ttft_ms") for r in rs]),
        "median_total_ms": med([r.get("total_ms") for r in rs]),
        "median_gen_tps": med([r.get("gen_tps") for r in rs]),
        "finish_reasons": [r.get("finish_reason") for r in rs],
    }

(run / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")

def f(v, n=2):
    return "-" if v is None else f"{v:.{n}f}"

md = [
    "# Ornith P3 — Reasoning Budget / Coding",
    "",
    f"Run: `{run}`",
    "",
    "Each profile gets the same 3 deterministic coding-reasoning exams.",
    "Exact PASS = 8/8 answers correct. Closure requires non-empty final content and finish_reason=stop.",
    "",
    "| Profile | budget | n | exact pass | closure | score med | reasoning tok med | final tok med | TTFT med ms | total med s | gen t/s med |",
    "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
]

for profile in order:
    if profile not in summary:
        continue
    x = summary[profile]

    md.append(
        f"| {profile} | {x['budget']} | {x['samples']} "
        f"| {x['exact_passes']}/{x['samples']} ({f(x['exact_pass_rate_pct'])}%) "
        f"| {x['closure_passes']}/{x['samples']} ({f(x['closure_rate_pct'])}%) "
        f"| {f(x['median_score'],1)}/8 "
        f"| {f(x['median_reasoning_tokens'],0)} "
        f"| {f(x['median_content_tokens'],0)} "
        f"| {f(x['median_ttft_ms'])} "
        f"| {f((x['median_total_ms']/1000) if x['median_total_ms'] else None)} "
        f"| {f(x['median_gen_tps'])} |"
    )

md += [
    "",
    "## Profiles",
    "",
    "- LOW: `thinking_budget_tokens=256`",
    "- MEDIUM: `thinking_budget_tokens=1024`",
    "- HIGH: `thinking_budget_tokens=2048`",
    "- All use `max_tokens=4096`, reasoning_format=auto, temperature=0.",
    "- P3 server is 1 × 32K, Q4/Q4 KV, FlashAttention ON, DFlash OFF.",
    "",
    "## What to choose",
    "",
    "Use the smallest profile that preserves exact task accuracy and closure.",
    "P3 intentionally does not auto-select a winner; inspect score, closure, latency, and token cost together.",
    "",
    "## Baseline restored",
    "",
    "- 3 × 256K",
    "- KV Q4/Q4",
    "- FlashAttention ON",
    "- DFlash OFF",
    "- cache-prompt ON",
    "- hard output 4096",
    "",
]

report = "\n".join(md) + "\n"
(run / "report.md").write_text(report)
print(report)
PY
}

# ----------------------------------------------------------------------------
# Manifest
# ----------------------------------------------------------------------------

make_manifest() {
    local image_id model_sha

    image_id="$(docker image inspect "$IMAGE" --format '{{.Id}}')"
    model_sha="$(sha256sum "$MODEL_DIR/$MODEL_FILE" | awk '{print $1}')"

    cat > "$RUN/manifest.json" <<EOF
{
  "created_utc": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "suite": "Ornith P3 reasoning budget / coding",
  "image": "$IMAGE",
  "image_id": "$image_id",
  "model_file": "$MODEL_FILE",
  "model_sha256": "$model_sha",
  "p3_ctx": $P3_CTX,
  "kv_k": "$KV_K",
  "kv_v": "$KV_V",
  "flash_attention": true,
  "dflash": false,
  "vision": false,
  "max_tokens": $OUTPUT_LIMIT,
  "rounds": $ROUNDS,
  "profiles": {
    "LOW": $LOW_BUDGET,
    "MEDIUM": $MEDIUM_BUDGET,
    "HIGH": $HIGH_BUDGET
  }
}
EOF
}

# ----------------------------------------------------------------------------
# Trap
# ----------------------------------------------------------------------------

cleanup_runtime() {
    stop_monitor || true
}

on_exit() {
    local rc=$?
    trap - EXIT INT TERM

    cleanup_runtime || true

    if [[ "$RESTORE_ON_EXIT" == "1" ]]; then
        restore_baseline || true
    fi

    exit "$rc"
}

# ----------------------------------------------------------------------------
# Main
# ----------------------------------------------------------------------------

run_all() {
    preflight_tools
    clean_old_clients

    RUN="$P3ROOT/p3-$(utc_stamp)"
    mkdir -p "$RUN/cases"

    echo "$RUN" > "$ROOT/LAST_P3"

    printf \
"profile\tbudget\tround\tscore\tresult\tformat\tclosure\tfinish\treasoning_tok\tfinal_tok\tdecoded\tttft_ms\ttotal_ms\tprompt_tps\tgen_tps\twrong\n" \
      > "$RUN/summary.tsv"

    : > "$RUN/results.jsonl"

    make_manifest
    write_helpers
    setup_tmux
    start_monitor

    RESTORE_ON_EXIT=1
    trap on_exit EXIT INT TERM

    run_profile "LOW" "$LOW_BUDGET"
    run_profile "MEDIUM" "$MEDIUM_BUDGET"
    run_profile "HIGH" "$HIGH_BUDGET"

    update_progress "ANALYSIS" "$ROUNDS" "done" -1

    make_report | tee "$RUN/report-console.txt"

    docker logs ornith > "$RUN/llama-last.log" 2>&1 || true

    RESTORE_ON_EXIT=0
    cleanup_runtime
    restore_baseline

    echo
    echo "============================================================================"
    echo "P3 TERMINADA — BASELINE RESTAURADO"
    echo "============================================================================"
    echo "Run:      $RUN"
    echo "TSV:      $RUN/summary.tsv"
    echo "JSON:     $RUN/summary.json"
    echo "Reporte:  $RUN/report.md"
}

results_view() {
    [[ -f "$ROOT/LAST_P3" ]] || die "No existe $ROOT/LAST_P3"

    local d
    d="$(cat "$ROOT/LAST_P3")"

    echo "$d"
    echo

    if [[ -f "$d/report.md" ]]; then
        cat "$d/report.md"
    else
        echo "P3 aún no termina."
        echo
        tail -n 12 "$d/summary.tsv" | column -t -s $'\t'
    fi
}

case "${1:-run}" in
    run)
        run_all
        ;;
    dashboard)
        dashboard
        ;;
    slots)
        slots_view
        ;;
    watch)
        watch -n 1 "$SELF slots"
        ;;
    results)
        results_view
        ;;
    restore)
        restore_baseline
        ;;
    *)
        echo "Uso:"
        echo "  $SELF run"
        echo "  $SELF dashboard"
        echo "  $SELF slots"
        echo "  $SELF watch"
        echo "  $SELF results"
        echo "  $SELF restore"
        exit 1
        ;;
esac
