#!/usr/bin/env bash
set -Eeuo pipefail

# ============================================================================
# Ornith P4 — Concurrency / slot geometry
#
# Final comparison:
#   G3-256K = 3 slots × 262144 ctx  (total 786432)  <-- current baseline
#   G4-192K = 4 slots × 196608 ctx  (total 786432)
#   G4-160K = 4 slots × 163840 ctx  (total 655360)
#
# Workload:
#   - 6 simultaneous coding/reasoning requests
#   - thinking_budget_tokens = 2048  (winner from P3-B)
#   - max_tokens = 4096
#   - 3 cold rounds per geometry
#   - identical prompts for the same round across all geometries
#   - strict 8/8 JSON scoring
#
# Measures:
#   - batch wall time
#   - max simultaneously busy slots
#   - pass rate
#   - median/p90 per-request latency
#   - median TTFT
#   - aggregate decoded tokens/s for the whole 6-request batch
#   - median per-request gen t/s
#   - avg GPU util, peak VRAM, peak power, max temperature
#
# IMPORTANT:
#   This is a direct llama.cpp API benchmark, not OpenCode.
#   That keeps P4 focused on scheduling/slot geometry while still using the
#   2048 reasoning budget selected by P3-B.
#
# Baseline restored on success OR failure:
#   3 × 256K, Q4/Q4, FA ON, DFlash OFF, cache ON, hard output 4096.
# ============================================================================

ROOT="${HOME}/bench/ornith"
P4ROOT="${ROOT}/p4"

IMAGE="lomelidev/llama.cpp:cu130-nccl-sse-faall-10974"
MODEL_DIR="${HOME}/models/ornith15-9b"
MODEL_FILE="Ornith-1.5-9B-Q4_K_L.gguf"

PORT=8120
MODEL_ALIAS="ornith-local"

KV_K="q4_0"
KV_V="q4_0"

OUTPUT_LIMIT=4096
THINKING_BUDGET=2048
BASELINE_OUTPUT=4096
CACHE_RAM=32768

AGENTS=6
ROUNDS=3

PROFILES=("G3-256K" "G4-192K" "G4-160K")
PROFILE_NP=(3 4 4)
PROFILE_CTX_TOTAL=(786432 786432 655360)
PROFILE_CTX_SLOT=(262144 196608 163840)

RUN=""
MON_PID=""
RESTORE_ON_EXIT=0

SELF="$(readlink -f "$0")"

mkdir -p "$P4ROOT"

die() {
    echo >&2
    echo "ERROR: $*" >&2
    exit 1
}

warn() {
    echo "WARNING: $*" >&2
}

utc_stamp() {
    date -u +%Y%m%dT%H%M%SZ
}

need_cmd() {
    command -v "$1" >/dev/null 2>&1 || die "Falta comando: $1"
}

preflight_tools() {
    local c
    for c in docker curl jq python3 nvidia-smi tmux ss sha256sum awk sed grep column; do
        need_cmd "$c"
    done

    [[ -f "$MODEL_DIR/$MODEL_FILE" ]] || die "No existe modelo: $MODEL_DIR/$MODEL_FILE"
    docker image inspect "$IMAGE" >/dev/null 2>&1 || die "No existe imagen Docker: $IMAGE"
    tmux has-session -t llama 2>/dev/null || die "No existe tmux session 'llama'"

    if [[ -d /opt/llama.cpp ]]; then
        grep -Rqs --include='*.cpp' --include='*.h' \
          'thinking_budget_tokens' \
          /opt/llama.cpp/tools/server \
          /opt/llama.cpp/common 2>/dev/null \
          || die "Tu llama.cpp local no contiene thinking_budget_tokens"
    else
        warn "/opt/llama.cpp no existe; no pude verificar thinking_budget_tokens en fuente local"
    fi

    return 0
}

# ----------------------------------------------------------------------------
# Cleanup contamination
# ----------------------------------------------------------------------------

clean_old_clients() {
    echo
    echo "== Limpiando clientes que puedan contaminar :8120 =="

    pkill -TERM -f 'opencode run.*local-ornith/ornith-local' 2>/dev/null || true
    pkill -TERM -f 'opencode\.exe serve --stdio --port 0' 2>/dev/null || true
    sleep 1
    pkill -KILL -f 'opencode run.*local-ornith/ornith-local' 2>/dev/null || true
    pkill -KILL -f 'opencode\.exe serve --stdio --port 0' 2>/dev/null || true

    mapfile -t PIDS < <(
        ss -tnp 2>/dev/null \
          | grep ':8120' \
          | grep -oE 'pid=[0-9]+' \
          | cut -d= -f2 \
          | sort -u \
          || true
    )

    if ((${#PIDS[@]})); then
        echo "Cortando clientes conectados a :8120: ${PIDS[*]}"
        kill -TERM "${PIDS[@]}" 2>/dev/null || true
        sleep 1

        local pid
        for pid in "${PIDS[@]}"; do
            kill -KILL "$pid" 2>/dev/null || true
        done
    fi

    if ss -tnp 2>/dev/null | grep -q ':8120'; then
        echo "Conexiones restantes:"
        ss -tnp 2>/dev/null | grep ':8120' || true
    else
        echo "Sin conexiones a :8120"
    fi

    return 0
}

# ----------------------------------------------------------------------------
# llama lifecycle
# ----------------------------------------------------------------------------

start_profile() {
    local profile="$1"
    local np="$2"
    local ctx_total="$3"
    local ctx_slot="$4"

    docker rm -f ornith >/dev/null 2>&1 || true

    docker run -d \
      --name ornith \
      --gpus "device=0" \
      --ipc=host \
      -p "${PORT}:8080" \
      -v "${MODEL_DIR}:/models:ro" \
      -e GGML_CUDA_GRAPH_OPT=1 \
      "$IMAGE" \
      -m "/models/${MODEL_FILE}" \
      -a "$MODEL_ALIAS" \
      -ngl 999 \
      -fa on \
      -ctk "$KV_K" \
      -ctv "$KV_V" \
      -c "$ctx_total" \
      -np "$np" \
      --no-kv-unified \
      --cont-batching \
      --no-mmproj \
      --n-predict "$OUTPUT_LIMIT" \
      -b 4096 \
      -ub 2048 \
      --cache-ram "$CACHE_RAM" \
      --cache-prompt \
      --ctx-checkpoints 4 \
      --jinja \
      --metrics \
      --host 0.0.0.0 \
      --port 8080 \
      --temp 0.0 \
      --top-p 1.0 \
      --top-k 0 \
      --min-p 0 \
      >/dev/null

    local ok=0
    local _
    for _ in $(seq 1 180); do
        if curl -fsS "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
            ok=1
            break
        fi

        if ! docker ps --format '{{.Names}}' | grep -qx ornith; then
            break
        fi

        sleep 1
    done

    if [[ "$ok" != "1" ]]; then
        docker logs ornith 2>&1 | tail -160 || true
        die "$profile: llama no arrancó"
    fi

    local logs
    logs="$(docker logs ornith 2>&1)"

    grep -Fq "n_slots = ${np}, n_ctx_slot = ${ctx_slot}" <<<"$logs" \
      || {
          echo "$logs" | grep -E 'n_slots|n_ctx_slot' | tail -10 || true
          die "$profile: no quedó ${np}×${ctx_slot}"
      }

    if grep -q 'no FlashAttention vector kernel' <<<"$logs"; then
        die "$profile: FA fallback detectado"
    fi

    if grep -Eqi 'cudaMalloc failed|out of memory|capping' <<<"$logs"; then
        die "$profile: OOM/capping detectado"
    fi

    docker inspect ornith --format '{{json .Args}}' \
      > "$RUN/${profile}-last-args.json" 2>/dev/null || true

    return 0
}

restore_baseline() {
    echo
    echo "== Restaurando baseline 3x256K / Q4Q4 / FA / cache ON / output=4096 =="

    docker rm -f ornith >/dev/null 2>&1 || true

    docker run -d \
      --name ornith \
      --gpus "device=0" \
      --ipc=host \
      -p "${PORT}:8080" \
      -v "${MODEL_DIR}:/models:ro" \
      -e GGML_CUDA_GRAPH_OPT=1 \
      "$IMAGE" \
      -m "/models/${MODEL_FILE}" \
      -a "$MODEL_ALIAS" \
      -ngl 999 \
      -fa on \
      -ctk "$KV_K" \
      -ctv "$KV_V" \
      -c 786432 \
      -np 3 \
      --no-kv-unified \
      --cont-batching \
      --no-mmproj \
      --n-predict "$BASELINE_OUTPUT" \
      -b 4096 \
      -ub 2048 \
      --cache-ram "$CACHE_RAM" \
      --cache-prompt \
      --ctx-checkpoints 4 \
      --jinja \
      --metrics \
      --host 0.0.0.0 \
      --port 8080 \
      --temp 0.6 \
      --top-p 0.95 \
      --top-k 20 \
      --min-p 0 \
      >/dev/null

    local ok=0
    local _
    for _ in $(seq 1 180); do
        if curl -fsS "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
            ok=1
            break
        fi
        sleep 1
    done

    [[ "$ok" == "1" ]] || die "No pude restaurar baseline"

    curl -s "http://127.0.0.1:${PORT}/slots" \
      | jq -r '.[] | "slot=\(.id) ctx=\(.n_ctx) busy=\(.is_processing) max=\(.params.max_tokens)"'

    return 0
}

# ----------------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------------

write_helpers() {
    cat > "$RUN/build_prompts.py" <<'PY'
#!/usr/bin/env python3
import json
import random
import sys
from pathlib import Path

root = Path(sys.argv[1])
rounds = int(sys.argv[2])
agents = int(sys.argv[3])

root.mkdir(parents=True, exist_ok=True)

def make_case(round_no: int, agent_no: int):
    # Each request is unique, but deterministic and exactly reproducible across
    # all geometry profiles.
    rng = random.Random(202609170000 + round_no * 100 + agent_no)

    seed = rng.randint(17, 57)
    x = seed
    for i in range(1, 9):
        if (x + i) % 4 == 0:
            x += i * 3
        elif x % 3 == 0:
            x //= 3
        elif x % 2 == 0:
            x //= 2
        else:
            x += i

    xs = [rng.randint(1, 25) for _ in range(10)]
    evens = [v for v in xs if v % 2 == 0]
    transformed = [v * 2 + i for i, v in enumerate(evens)]
    q6 = sum(transformed)

    arr = [rng.randint(2, 30) for _ in range(9)]
    q8 = sum(v if i % 2 == 0 else -v for i, v in enumerate(arr))

    expected = {
        "q1": str(x),
        "q2": "B",
        "q3": "C",
        "q4": "C",
        "q5": "A,E,B,C,D",
        "q6": str(q6),
        "q7": "B",
        "q8": str(q8),
    }

    # Add deterministic "repo context" so this looks more like a coding-agent
    # request than the tiny P3 prompt. The issue itself is at the end.
    ctx = []
    for i in range(1, 90):
        a = rng.randint(3, 99)
        b = rng.randint(3, 99)
        ctx.append(
            f"// repo/module_{agent_no:02d}_{i:03d}.ts\n"
            f"export function helper_{agent_no}_{i}(x: number): number {{\n"
            f"  const a = {a};\n"
            f"  const b = {b};\n"
            f"  return ((x * a) + b + {i}) % 10007;\n"
            f"}}\n"
        )

    prompt = f"""You are working on coding issue P4-R{round_no}-A{agent_no}.

The following repository context is synthetic. Read it, then solve the eight
questions at the end.

{"".join(ctx)}

--- ISSUE / FINAL QUESTIONS ---

Think internally, but your FINAL response MUST be exactly one JSON object.
No markdown. No prose before or after the JSON.
Exactly these keys: q1,q2,q3,q4,q5,q6,q7,q8
Every value must be a JSON string.

Q1 — TypeScript trace

let x = {seed};
for (let i = 1; i <= 8; i++) {{
  if ((x + i) % 4 === 0) {{
    x += i * 3;
  }} else if (x % 3 === 0) {{
    x = Math.floor(x / 3);
  }} else if (x % 2 === 0) {{
    x = Math.floor(x / 2);
  }} else {{
    x += i;
  }}
}}
Return final x as decimal string.

Q2 — Go cancellation

for job := range jobs {{
    select {{
    case <-ctx.Done():
        return
    default:
    }}
    out <- job.Execute()
}}

Which patch directly prevents a worker from remaining blocked forever on the
result send after cancellation?

A) close(out) before each send
B) r := job.Execute(); select {{ case out <- r: case <-ctx.Done(): return }}
C) one more ctx check before Execute(), then keep blocking send
D) buffer out with capacity 1

Return the letter.

Q3 — PostgreSQL latest paid order per user

Need one row for every user, including users with no paid orders.
Latest paid order wins, with greatest id as tie-breaker.

A) INNER JOIN + GROUP BY
B) LEFT JOIN all paid orders + global LIMIT 1
C) LEFT JOIN LATERAL (
     SELECT id FROM orders
     WHERE user_id=u.id AND status='paid'
     ORDER BY created_at DESC, id DESC LIMIT 1
   ) o ON true
D) LEFT JOIN + max(order_id)

Return the letter.

Q4 — Sliding-window counter

Increment() expires ring buckets, but GetCount() only returns runningSum.
After an idle period longer than the window, stale events remain visible.

A) atomics solve it
B) double bucket count
C) GetCount() must advance/expire buckets with the current clock first
D) reset after each Increment

Return the letter.

Q5 — JavaScript event loop

console.log("A");
Promise.resolve().then(() => console.log("B"));
queueMicrotask(() => console.log("C"));
setTimeout(() => console.log("D"), 0);
console.log("E");

Return one comma-separated string, no spaces.

Q6 — TypeScript transform

const xs = {json.dumps(xs)};
const y = xs
  .filter(v => v % 2 === 0)
  .map((v, i) => v * 2 + i)
  .reduce((a, b) => a + b, 0);

Return y as decimal string.

Q7 — Lock ordering

F locks A then B.
G locks B then A.
Both can run concurrently.

A) sleep before G
B) enforce one global lock order everywhere
C) RWMutex, same opposite order
D) increase GOMAXPROCS

Return the letter.

Q8 — deterministic trace

const a = {json.dumps(arr)};
let z = 0;
for (let i = 0; i < a.length; i++) {{
  if (i % 2 === 0) z += a[i];
  else z -= a[i];
}}

Return z as decimal string.

FINAL RESPONSE SHAPE:
{{"q1":"...","q2":"...","q3":"...","q4":"...","q5":"...","q6":"...","q7":"...","q8":"..."}}
"""

    return prompt, expected, {
        "round": round_no,
        "agent": agent_no,
        "seed": seed,
        "xs": xs,
        "arr": arr,
        "expected": expected,
    }

for round_no in range(1, rounds + 1):
    rd = root / f"round-{round_no:02d}"
    rd.mkdir(parents=True, exist_ok=True)

    for agent_no in range(1, agents + 1):
        prompt, expected, meta = make_case(round_no, agent_no)
        (rd / f"agent-{agent_no:02d}.prompt.txt").write_text(prompt)
        (rd / f"agent-{agent_no:02d}.expected.json").write_text(
            json.dumps(meta, indent=2) + "\n"
        )
PY

    cat > "$RUN/chat_client.py" <<'PY'
#!/usr/bin/env python3
import http.client
import json
import sys
import time
from pathlib import Path

prompt_path = Path(sys.argv[1])
sse_path = Path(sys.argv[2])
response_path = Path(sys.argv[3])
port = int(sys.argv[4])
output_limit = int(sys.argv[5])
thinking_budget = int(sys.argv[6])
seed = int(sys.argv[7])

payload = {
    "model": "ornith-local",
    "messages": [{"role": "user", "content": prompt_path.read_text()}],
    "stream": True,
    "max_tokens": output_limit,
    "temperature": 0.0,
    "top_p": 1.0,
    "top_k": 0,
    "seed": seed,
    "reasoning_format": "auto",
    "thinking_budget_tokens": thinking_budget,
}

body = json.dumps(payload).encode()

conn = http.client.HTTPConnection("127.0.0.1", port, timeout=1800)

t0 = time.monotonic()
epoch0 = time.time()

conn.request(
    "POST",
    "/v1/chat/completions",
    body=body,
    headers={
        "Content-Type": "application/json",
        "Content-Length": str(len(body)),
        "Connection": "close",
    },
)

resp = conn.getresponse()

first_delta = None
content_parts = []
reasoning_parts = []
finish_reason = None
last_obj = None

with sse_path.open("wb") as raw:
    while True:
        line = resp.readline()
        if not line:
            break

        raw.write(line)
        raw.flush()

        stripped = line.strip()
        if not stripped.startswith(b"data:"):
            continue

        data = stripped[5:].strip()
        if not data or data == b"[DONE]":
            continue

        try:
            obj = json.loads(data)
        except Exception:
            continue

        last_obj = obj
        choices = obj.get("choices") or []
        if not choices:
            continue

        choice = choices[0]
        delta = choice.get("delta") or {}

        content = delta.get("content")
        reasoning = delta.get("reasoning_content")

        if content:
            if first_delta is None:
                first_delta = time.monotonic()
            content_parts.append(content)

        if reasoning:
            if first_delta is None:
                first_delta = time.monotonic()
            reasoning_parts.append(reasoning)

        if choice.get("finish_reason") is not None:
            finish_reason = choice.get("finish_reason")

t1 = time.monotonic()

result = {
    "http_status": resp.status,
    "start_epoch": epoch0,
    "end_epoch": time.time(),
    "ttft_ms": None if first_delta is None else (first_delta - t0) * 1000.0,
    "total_ms": (t1 - t0) * 1000.0,
    "finish_reason": finish_reason,
    "content": "".join(content_parts),
    "reasoning_content": "".join(reasoning_parts),
    "last_chunk": last_obj,
}

response_path.write_text(json.dumps(result, indent=2) + "\n")
conn.close()

if resp.status < 200 or resp.status >= 300:
    sys.exit(2)
PY

    cat > "$RUN/score.py" <<'PY'
#!/usr/bin/env python3
import http.client
import json
import sys
from pathlib import Path

response_path = Path(sys.argv[1])
expected_path = Path(sys.argv[2])
result_path = Path(sys.argv[3])
port = int(sys.argv[4])

response = json.loads(response_path.read_text())
expected = json.loads(expected_path.read_text())["expected"]

content = response.get("content") or ""
reasoning = response.get("reasoning_content") or ""
stripped = content.strip()

parsed = None
try:
    parsed = json.loads(stripped)
except Exception:
    pass

exact_keys = (
    isinstance(parsed, dict)
    and list(parsed.keys()) == list(expected.keys())
)

answers = {}
if isinstance(parsed, dict):
    answers = {k: str(v).strip() for k, v in parsed.items()}

per_question = {
    k: answers.get(k) == str(v)
    for k, v in expected.items()
}

score = sum(per_question.values())
strict_format_ok = isinstance(parsed, dict) and exact_keys
closure_ok = response.get("finish_reason") == "stop" and bool(stripped)
strict_pass = strict_format_ok and closure_ok and score == len(expected)

def tok_count(text):
    if not text:
        return 0

    body = json.dumps({"content": text}).encode()
    conn = http.client.HTTPConnection("127.0.0.1", port, timeout=60)
    conn.request(
        "POST",
        "/tokenize",
        body=body,
        headers={
            "Content-Type": "application/json",
            "Content-Length": str(len(body)),
        },
    )
    r = conn.getresponse()
    raw = r.read()
    conn.close()

    if not (200 <= r.status < 300):
        return None

    obj = json.loads(raw)
    tokens = obj.get("tokens")

    if isinstance(tokens, list):
        return len(tokens)
    if isinstance(tokens, int):
        return tokens
    return None

reasoning_tokens = tok_count(reasoning)
content_tokens = tok_count(content)

last = response.get("last_chunk") or {}
timings = last.get("timings") or {}
usage = last.get("usage") or {}

decoded = timings.get("predicted_n")
if not isinstance(decoded, (int, float)):
    decoded = usage.get("completion_tokens")
if not isinstance(decoded, (int, float)):
    if isinstance(reasoning_tokens, int) and isinstance(content_tokens, int):
        decoded = reasoning_tokens + content_tokens

result = {
    "score": score,
    "max_score": len(expected),
    "strict_pass": strict_pass,
    "strict_format_ok": strict_format_ok,
    "closure_ok": closure_ok,
    "finish_reason": response.get("finish_reason"),
    "reasoning_tokens": reasoning_tokens,
    "content_tokens": content_tokens,
    "decoded_tokens": decoded,
    "ttft_ms": response.get("ttft_ms"),
    "total_ms": response.get("total_ms"),
    "prompt_tps": timings.get("prompt_per_second"),
    "gen_tps": timings.get("predicted_per_second"),
    "per_question": per_question,
    "answers": answers,
}

result_path.write_text(json.dumps(result, indent=2) + "\n")
PY

    cat > "$RUN/sample_batch.py" <<'PY'
#!/usr/bin/env python3
import json
import os
import subprocess
import sys
import time
import urllib.request
from pathlib import Path

marker_pid = int(sys.argv[1])
port = int(sys.argv[2])
jsonl_path = Path(sys.argv[3])
summary_path = Path(sys.argv[4])

def alive(pid):
    try:
        os.kill(pid, 0)
        return True
    except OSError:
        return False

def slots():
    try:
        with urllib.request.urlopen(
            f"http://127.0.0.1:{port}/slots",
            timeout=0.5,
        ) as r:
            return json.loads(r.read())
    except Exception:
        return []

def gpu():
    try:
        raw = subprocess.check_output(
            [
                "nvidia-smi", "-i", "0",
                "--query-gpu=utilization.gpu,memory.used,memory.free,power.draw,temperature.gpu",
                "--format=csv,noheader,nounits",
            ],
            text=True,
            timeout=2,
        ).strip()

        parts = [x.strip() for x in raw.split(",")]
        return {
            "gpu_util": float(parts[0]),
            "gpu_used_mib": float(parts[1]),
            "gpu_free_mib": float(parts[2]),
            "power_w": float(parts[3]),
            "temp_c": float(parts[4]),
        }
    except Exception:
        return {
            "gpu_util": None,
            "gpu_used_mib": None,
            "gpu_free_mib": None,
            "power_w": None,
            "temp_c": None,
        }

rows = []

with jsonl_path.open("w") as out:
    while alive(marker_pid):
        ss = slots()
        busy = sum(1 for s in ss if s.get("is_processing"))
        g = gpu()

        rec = {
            "ts": time.time(),
            "busy": busy,
            **g,
        }

        rows.append(rec)
        out.write(json.dumps(rec) + "\n")
        out.flush()

        time.sleep(0.25)

def vals(key):
    return [
        float(r[key])
        for r in rows
        if isinstance(r.get(key), (int, float))
    ]

gpu_utils = vals("gpu_util")
used = vals("gpu_used_mib")
power = vals("power_w")
temp = vals("temp_c")

summary = {
    "samples": len(rows),
    "max_busy": max([r["busy"] for r in rows], default=0),
    "avg_gpu_util": sum(gpu_utils) / len(gpu_utils) if gpu_utils else None,
    "peak_gpu_used_mib": max(used) if used else None,
    "peak_power_w": max(power) if power else None,
    "max_temp_c": max(temp) if temp else None,
}

summary_path.write_text(json.dumps(summary, indent=2) + "\n")
PY

    chmod +x \
      "$RUN/build_prompts.py" \
      "$RUN/chat_client.py" \
      "$RUN/score.py" \
      "$RUN/sample_batch.py"
}

# ----------------------------------------------------------------------------
# tmux / progress
# ----------------------------------------------------------------------------

ensure_window() {
    local name="$1"
    if ! tmux list-windows -t llama -F '#{window_name}' | grep -qx "$name"; then
        tmux new-window -d -t llama -n "$name"
    fi
}

setup_tmux() {
    local w
    for w in Log Check Test Monitor OpenCode; do
        ensure_window "$w"
    done

    tmux send-keys -t llama:Log C-c
    tmux send-keys -t llama:Log \
      'while true; do clear; docker logs -f ornith 2>&1; sleep 1; done' C-m

    tmux send-keys -t llama:Check C-c
    tmux send-keys -t llama:Check \
      "watch -n 1 '$SELF dashboard'" C-m

    tmux send-keys -t llama:OpenCode C-c
    tmux send-keys -t llama:OpenCode \
      "watch -n 1 '$SELF slots'" C-m
}

update_progress() {
    local profile="$1"
    local round="$2"
    local phase="$3"
    local done="$4"

    python3 - "$RUN/progress.json" "$profile" "$round" "$phase" "$done" "$AGENTS" <<'PY'
import json, os, sys, time

path, profile, rnd, phase, done, total = sys.argv[1:]

obj = {
    "profile": profile,
    "round": int(rnd),
    "phase": phase,
    "completed": int(done),
    "total": int(total),
    "updated_epoch": time.time(),
}

tmp = path + ".tmp"
with open(tmp, "w") as f:
    json.dump(obj, f)
os.replace(tmp, path)
PY
}

dashboard() {
    [[ -f "$ROOT/LAST_P4" ]] || {
        echo "No hay LAST_P4 todavía."
        return
    }

    local d
    d="$(cat "$ROOT/LAST_P4")"

    clear
    echo "ORNITH P4 — CONCURRENCY / SLOT GEOMETRY"
    echo "============================================================================"
    echo "$d"
    echo

    if [[ -f "$d/progress.json" ]]; then
        jq -r '
          "Profile   : \(.profile)
Round     : \(.round)
Phase     : \(.phase)
Completed : \(.completed)/\(.total)"
        ' "$d/progress.json"
    else
        echo "Esperando..."
    fi

    echo
    echo "BATCHES"
    echo "============================================================================"

    if [[ -s "$d/batches.tsv" ]]; then
        tail -n 12 "$d/batches.tsv" | column -t -s $'\t'
    fi

    echo
    echo "ÚLTIMOS REQUESTS"
    echo "============================================================================"

    if [[ -s "$d/requests.tsv" ]]; then
        tail -n 10 "$d/requests.tsv" | column -t -s $'\t'
    fi

    echo
    echo "GPU0"
    echo "============================================================================"

    nvidia-smi -i 0 \
      --query-gpu=utilization.gpu,memory.used,memory.free,power.draw,temperature.gpu \
      --format=csv,noheader
}

slots_view() {
    curl -s "http://127.0.0.1:${PORT}/slots" \
      | jq -r '
        .[] |
        "SLOT \(.id) task=\(.id_task) busy=\(.is_processing)
ctx=\(.n_ctx)
prompt=\(.n_prompt_tokens)
processed=\(.n_prompt_tokens_processed)
cache=\(.n_prompt_tokens_cache)
decoded=\(.next_token[0].n_decoded // 0)
max_tokens=\(.params.max_tokens)
---------------------------------------------"
      '
}

# ----------------------------------------------------------------------------
# One six-agent batch
# ----------------------------------------------------------------------------

run_batch() {
    local profile="$1"
    local np="$2"
    local ctx_total="$3"
    local ctx_slot="$4"
    local round="$5"

    local batch_dir="$RUN/cases/$profile/round-$(printf '%02d' "$round")"
    mkdir -p "$batch_dir"

    update_progress "$profile" "$round" "startup" 0

    # Fresh server per batch. No prompt-cache carryover between geometries/rounds.
    start_profile "$profile" "$np" "$ctx_total" "$ctx_slot"

    local prompt_dir="$RUN/prompts/round-$(printf '%02d' "$round")"

    update_progress "$profile" "$round" "launching" 0

    local marker_pid
    sleep 3600 &
    marker_pid=$!

    python3 "$RUN/sample_batch.py" \
      "$marker_pid" \
      "$PORT" \
      "$batch_dir/batch-samples.jsonl" \
      "$batch_dir/batch-system.json" &
    local sampler_pid=$!

    local start_ns end_ns
    start_ns="$(date +%s%N)"

    local -a pids=()
    local -a rcs=()
    local a

    for a in $(seq 1 "$AGENTS"); do
        local agent_dir="$batch_dir/agent-$(printf '%02d' "$a")"
        mkdir -p "$agent_dir"

        python3 "$RUN/chat_client.py" \
          "$prompt_dir/agent-$(printf '%02d' "$a").prompt.txt" \
          "$agent_dir/response.sse" \
          "$agent_dir/response.json" \
          "$PORT" \
          "$OUTPUT_LIMIT" \
          "$THINKING_BUDGET" \
          "$((1000 + round * 100 + a))" \
          > "$agent_dir/client.stdout" \
          2> "$agent_dir/client.stderr" &

        pids+=("$!")
    done

    local failures=0
    local completed=0
    local i

    for i in "${!pids[@]}"; do
        local rc=0

        wait "${pids[$i]}" || rc=$?
        rcs+=("$rc")

        completed=$((completed + 1))
        update_progress "$profile" "$round" "running" "$completed"

        if [[ "$rc" != "0" ]]; then
            failures=$((failures + 1))
        fi
    done

    end_ns="$(date +%s%N)"

    kill "$marker_pid" 2>/dev/null || true
    wait "$marker_pid" 2>/dev/null || true
    wait "$sampler_pid" 2>/dev/null || true

    local wall_ms
    wall_ms="$(awk -v s="$start_ns" -v e="$end_ns" 'BEGIN { printf "%.3f", (e-s)/1000000.0 }')"

    docker logs ornith > "$batch_dir/llama.log" 2>&1 || true

    local capping fa_warn oom
    capping="$(grep -ci 'capping' "$batch_dir/llama.log" || true)"
    fa_warn="$(grep -c 'no FlashAttention vector kernel' "$batch_dir/llama.log" || true)"
    oom="$(grep -Eci 'cudaMalloc failed|out of memory' "$batch_dir/llama.log" || true)"

    [[ "$capping" == "0" ]] || die "$profile round=$round tuvo capping"
    [[ "$fa_warn" == "0" ]] || die "$profile round=$round tuvo FA fallback"
    [[ "$oom" == "0" ]] || die "$profile round=$round tuvo OOM"

    # Score all completed responses. A failed HTTP client remains a failed request.
    for a in $(seq 1 "$AGENTS"); do
        local agent_dir="$batch_dir/agent-$(printf '%02d' "$a")"

        if [[ -f "$agent_dir/response.json" ]]; then
            python3 "$RUN/score.py" \
              "$agent_dir/response.json" \
              "$prompt_dir/agent-$(printf '%02d' "$a").expected.json" \
              "$agent_dir/scored.json" \
              "$PORT"
        else
            cat > "$agent_dir/scored.json" <<EOF
{
  "score": 0,
  "max_score": 8,
  "strict_pass": false,
  "strict_format_ok": false,
  "closure_ok": false,
  "finish_reason": "client_error",
  "reasoning_tokens": null,
  "content_tokens": null,
  "decoded_tokens": 0,
  "ttft_ms": null,
  "total_ms": null,
  "prompt_tps": null,
  "gen_tps": null,
  "per_question": {}
}
EOF
        fi
    done

    python3 - \
      "$profile" "$np" "$ctx_slot" "$round" "$wall_ms" \
      "$batch_dir" \
      "$RUN/requests.jsonl" \
      "$RUN/requests.tsv" \
      "$RUN/batches.jsonl" \
      "$RUN/batches.tsv" <<'PY'
import json
import math
import statistics
import sys
from pathlib import Path

(
    profile, np, ctx_slot, rnd, wall_ms,
    batch_dir,
    req_jsonl, req_tsv,
    batch_jsonl, batch_tsv,
) = sys.argv[1:]

np = int(np)
ctx_slot = int(ctx_slot)
rnd = int(rnd)
wall_ms = float(wall_ms)
batch_dir = Path(batch_dir)

rows = []

for p in sorted(batch_dir.glob("agent-*/scored.json")):
    agent = int(p.parent.name.split("-")[-1])
    s = json.loads(p.read_text())

    row = {
        "profile": profile,
        "round": rnd,
        "agent": agent,
        "score": s.get("score"),
        "max_score": s.get("max_score"),
        "strict_pass": bool(s.get("strict_pass")),
        "format_ok": bool(s.get("strict_format_ok")),
        "closure_ok": bool(s.get("closure_ok")),
        "finish_reason": s.get("finish_reason"),
        "reasoning_tokens": s.get("reasoning_tokens"),
        "content_tokens": s.get("content_tokens"),
        "decoded_tokens": s.get("decoded_tokens"),
        "ttft_ms": s.get("ttft_ms"),
        "total_ms": s.get("total_ms"),
        "prompt_tps": s.get("prompt_tps"),
        "gen_tps": s.get("gen_tps"),
    }

    rows.append(row)

    with open(req_jsonl, "a") as f:
        f.write(json.dumps(row) + "\n")

    with open(req_tsv, "a") as f:
        f.write("\t".join([
            profile,
            str(rnd),
            str(agent),
            f'{row["score"]}/{row["max_score"]}',
            "PASS" if row["strict_pass"] else "FAIL",
            "yes" if row["format_ok"] else "no",
            "yes" if row["closure_ok"] else "no",
            str(row["finish_reason"] or ""),
            "" if row["reasoning_tokens"] is None else str(row["reasoning_tokens"]),
            "" if row["content_tokens"] is None else str(row["content_tokens"]),
            "" if row["decoded_tokens"] is None else str(row["decoded_tokens"]),
            "" if row["ttft_ms"] is None else f'{row["ttft_ms"]:.2f}',
            "" if row["total_ms"] is None else f'{row["total_ms"]:.2f}',
            "" if row["gen_tps"] is None else f'{row["gen_tps"]:.2f}',
        ]) + "\n")

system_path = batch_dir / "batch-system.json"
system = json.loads(system_path.read_text()) if system_path.exists() else {}

def nums(key):
    return [
        float(r[key])
        for r in rows
        if isinstance(r.get(key), (int, float))
    ]

def med(key):
    xs = nums(key)
    return statistics.median(xs) if xs else None

def percentile(key, q):
    xs = sorted(nums(key))
    if not xs:
        return None
    if len(xs) == 1:
        return xs[0]
    pos = (len(xs) - 1) * q
    lo = int(math.floor(pos))
    hi = min(lo + 1, len(xs) - 1)
    frac = pos - lo
    return xs[lo] * (1 - frac) + xs[hi] * frac

decoded_total = sum(nums("decoded_tokens"))
agg_tps = decoded_total / (wall_ms / 1000.0) if wall_ms > 0 else None

batch = {
    "profile": profile,
    "slots": np,
    "ctx_slot": ctx_slot,
    "round": rnd,
    "requests": len(rows),
    "passes": sum(1 for r in rows if r["strict_pass"]),
    "format_passes": sum(1 for r in rows if r["format_ok"]),
    "closure_passes": sum(1 for r in rows if r["closure_ok"]),
    "batch_wall_ms": wall_ms,
    "median_total_ms": med("total_ms"),
    "p90_total_ms": percentile("total_ms", 0.90),
    "median_ttft_ms": med("ttft_ms"),
    "decoded_total": decoded_total,
    "aggregate_decoded_tps": agg_tps,
    "median_gen_tps": med("gen_tps"),
    "median_reasoning_tokens": med("reasoning_tokens"),
    "max_busy": system.get("max_busy"),
    "avg_gpu_util": system.get("avg_gpu_util"),
    "peak_gpu_used_mib": system.get("peak_gpu_used_mib"),
    "peak_power_w": system.get("peak_power_w"),
    "max_temp_c": system.get("max_temp_c"),
}

with open(batch_jsonl, "a") as f:
    f.write(json.dumps(batch) + "\n")

def fmt(v, n=2):
    return "" if v is None else f"{v:.{n}f}"

with open(batch_tsv, "a") as f:
    f.write("\t".join([
        profile,
        str(np),
        str(ctx_slot),
        str(rnd),
        f'{batch["passes"]}/{batch["requests"]}',
        str(batch["max_busy"] if batch["max_busy"] is not None else ""),
        fmt(batch["batch_wall_ms"]),
        fmt(batch["median_total_ms"]),
        fmt(batch["p90_total_ms"]),
        fmt(batch["median_ttft_ms"]),
        fmt(batch["aggregate_decoded_tps"]),
        fmt(batch["median_gen_tps"]),
        fmt(batch["avg_gpu_util"]),
        fmt(batch["peak_gpu_used_mib"], 0),
        fmt(batch["peak_power_w"]),
        fmt(batch["max_temp_c"], 0),
    ]) + "\n")

print(json.dumps(batch, indent=2))
PY

    update_progress "$profile" "$round" "done" "$AGENTS"

    echo
    echo "BATCH RESULT $profile round=$round"
    tail -n 2 "$RUN/batches.tsv" | column -t -s $'\t'

    if [[ "$failures" != "0" ]]; then
        warn "$profile round=$round tuvo $failures clientes HTTP con error"
    fi

    return 0
}

# ----------------------------------------------------------------------------
# Final report
# ----------------------------------------------------------------------------

make_report() {
    python3 - "$RUN" <<'PY'
from pathlib import Path
import json
import statistics
import sys

run = Path(sys.argv[1])

batches = [
    json.loads(x)
    for x in (run / "batches.jsonl").read_text().splitlines()
    if x.strip()
]

requests = [
    json.loads(x)
    for x in (run / "requests.jsonl").read_text().splitlines()
    if x.strip()
]

order = ["G3-256K", "G4-192K", "G4-160K"]

def med(xs):
    xs = [float(x) for x in xs if isinstance(x, (int, float))]
    return statistics.median(xs) if xs else None

summary = {}

for profile in order:
    bs = [b for b in batches if b["profile"] == profile]
    rs = [r for r in requests if r["profile"] == profile]

    if not bs:
        continue

    summary[profile] = {
        "slots": bs[0]["slots"],
        "ctx_slot": bs[0]["ctx_slot"],
        "rounds": len(bs),
        "requests": len(rs),
        "strict_passes": sum(1 for r in rs if r.get("strict_pass")),
        "strict_pass_rate_pct": (
            100.0 * sum(1 for r in rs if r.get("strict_pass")) / len(rs)
            if rs else None
        ),
        "median_batch_wall_ms": med([b.get("batch_wall_ms") for b in bs]),
        "median_request_total_ms": med([r.get("total_ms") for r in rs]),
        "median_ttft_ms": med([r.get("ttft_ms") for r in rs]),
        "median_aggregate_decoded_tps": med([b.get("aggregate_decoded_tps") for b in bs]),
        "median_request_gen_tps": med([r.get("gen_tps") for r in rs]),
        "median_max_busy": med([b.get("max_busy") for b in bs]),
        "median_avg_gpu_util": med([b.get("avg_gpu_util") for b in bs]),
        "peak_gpu_used_mib": max(
            [b.get("peak_gpu_used_mib") for b in bs if isinstance(b.get("peak_gpu_used_mib"), (int, float))],
            default=None,
        ),
        "peak_power_w": max(
            [b.get("peak_power_w") for b in bs if isinstance(b.get("peak_power_w"), (int, float))],
            default=None,
        ),
        "max_temp_c": max(
            [b.get("max_temp_c") for b in bs if isinstance(b.get("max_temp_c"), (int, float))],
            default=None,
        ),
    }

(run / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")

def f(v, n=2):
    return "-" if v is None else f"{v:.{n}f}"

md = [
    "# Ornith P4 — Concurrency / Slot Geometry",
    "",
    f"Run: `{run}`",
    "",
    "Workload: 6 simultaneous coding/reasoning requests, thinking_budget_tokens=2048, max_tokens=4096, 3 cold rounds per geometry.",
    "",
    "| Geometry | slots | ctx/slot | requests | strict pass | batch wall med s | req med s | TTFT med ms | aggregate tok/s | req gen t/s | max busy | avg GPU % | peak VRAM MiB |",
    "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
]

for profile in order:
    if profile not in summary:
        continue

    x = summary[profile]

    md.append(
        f"| {profile} | {x['slots']} | {x['ctx_slot']} | {x['requests']} "
        f"| {x['strict_passes']}/{x['requests']} ({f(x['strict_pass_rate_pct'])}%) "
        f"| {f((x['median_batch_wall_ms']/1000) if x['median_batch_wall_ms'] else None)} "
        f"| {f((x['median_request_total_ms']/1000) if x['median_request_total_ms'] else None)} "
        f"| {f(x['median_ttft_ms'])} "
        f"| {f(x['median_aggregate_decoded_tps'])} "
        f"| {f(x['median_request_gen_tps'])} "
        f"| {f(x['median_max_busy'],0)} "
        f"| {f(x['median_avg_gpu_util'])} "
        f"| {f(x['peak_gpu_used_mib'],0)} |"
    )

md += [
    "",
    "## Geometry definitions",
    "",
    "- `G3-256K`: 3 × 262144 = 786432 total context.",
    "- `G4-192K`: 4 × 196608 = 786432 total context.",
    "- `G4-160K`: 4 × 163840 = 655360 total context.",
    "",
    "## Notes",
    "",
    "- Every round starts from a fresh llama-server, so no prompt-cache carryover between rounds.",
    "- All profiles see the exact same prompts for a given round.",
    "- `aggregate tok/s` is total decoded tokens from the six requests divided by batch wall time.",
    "- This P4 isolates llama.cpp scheduling/slot geometry; it is not an OpenCode integration benchmark.",
    "",
    "## Baseline restored",
    "",
    "- 3 × 256K",
    "- KV Q4/Q4",
    "- FlashAttention ON",
    "- DFlash OFF",
    "- cache-prompt ON",
    "- hard output 4096",
    "",
]

report = "\n".join(md) + "\n"
(run / "report.md").write_text(report)
print(report)
PY
}

# ----------------------------------------------------------------------------
# Manifest / trap / main
# ----------------------------------------------------------------------------

make_manifest() {
    local image_id model_sha

    image_id="$(docker image inspect "$IMAGE" --format '{{.Id}}')"
    model_sha="$(sha256sum "$MODEL_DIR/$MODEL_FILE" | awk '{print $1}')"

    cat > "$RUN/manifest.json" <<EOF
{
  "created_utc": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "suite": "Ornith P4 concurrency / slot geometry",
  "image": "$IMAGE",
  "image_id": "$image_id",
  "model_file": "$MODEL_FILE",
  "model_sha256": "$model_sha",
  "kv_k": "$KV_K",
  "kv_v": "$KV_V",
  "flash_attention": true,
  "dflash": false,
  "vision": false,
  "cache_prompt": true,
  "cache_ram_mb": $CACHE_RAM,
  "thinking_budget_tokens": $THINKING_BUDGET,
  "max_tokens": $OUTPUT_LIMIT,
  "agents_per_batch": $AGENTS,
  "rounds_per_geometry": $ROUNDS,
  "geometries": {
    "G3-256K": {"slots": 3, "ctx_slot": 262144, "ctx_total": 786432},
    "G4-192K": {"slots": 4, "ctx_slot": 196608, "ctx_total": 786432},
    "G4-160K": {"slots": 4, "ctx_slot": 163840, "ctx_total": 655360}
  }
}
EOF
}

cleanup_runtime() {
    return 0
}

on_exit() {
    local rc=$?
    trap - EXIT INT TERM

    if [[ "$RESTORE_ON_EXIT" == "1" ]]; then
        restore_baseline || true
    fi

    exit "$rc"
}

run_all() {
    preflight_tools
    clean_old_clients

    RUN="$P4ROOT/p4-$(utc_stamp)"
    mkdir -p "$RUN"/{cases,prompts}

    echo "$RUN" > "$ROOT/LAST_P4"

    printf \
"profile\tround\tagent\tscore\tresult\tformat\tclosure\tfinish\treasoning_tok\tfinal_tok\tdecoded\tttft_ms\ttotal_ms\tgen_tps\n" \
      > "$RUN/requests.tsv"

    printf \
"profile\tslots\tctx_slot\tround\tpasses\tmax_busy\tbatch_wall_ms\trequest_med_ms\trequest_p90_ms\tttft_med_ms\tagg_decoded_tps\trequest_gen_tps\tavg_gpu_pct\tpeak_vram_mib\tpeak_power_w\tmax_temp_c\n" \
      > "$RUN/batches.tsv"

    : > "$RUN/requests.jsonl"
    : > "$RUN/batches.jsonl"

    make_manifest
    write_helpers

    python3 "$RUN/build_prompts.py" \
      "$RUN/prompts" \
      "$ROUNDS" \
      "$AGENTS"

    setup_tmux

    RESTORE_ON_EXIT=1
    trap on_exit EXIT INT TERM

    local i r

    for i in "${!PROFILES[@]}"; do
        local profile="${PROFILES[$i]}"
        local np="${PROFILE_NP[$i]}"
        local ctx_total="${PROFILE_CTX_TOTAL[$i]}"
        local ctx_slot="${PROFILE_CTX_SLOT[$i]}"

        echo
        echo "============================================================================"
        echo "P4 $profile — ${np} slots × ${ctx_slot}"
        echo "============================================================================"

        for r in $(seq 1 "$ROUNDS"); do
            echo
            echo "-- $profile round $r/$ROUNDS: 6 concurrent requests --"
            run_batch "$profile" "$np" "$ctx_total" "$ctx_slot" "$r"
        done
    done

    update_progress "ANALYSIS" "$ROUNDS" "done" "$AGENTS"

    make_report | tee "$RUN/report-console.txt"

    RESTORE_ON_EXIT=0
    restore_baseline

    echo
    echo "============================================================================"
    echo "P4 TERMINADA — BASELINE RESTAURADO"
    echo "============================================================================"
    echo "Run:       $RUN"
    echo "Requests:  $RUN/requests.tsv"
    echo "Batches:   $RUN/batches.tsv"
    echo "JSON:      $RUN/summary.json"
    echo "Reporte:   $RUN/report.md"
}

results_view() {
    [[ -f "$ROOT/LAST_P4" ]] || die "No existe $ROOT/LAST_P4"

    local d
    d="$(cat "$ROOT/LAST_P4")"

    echo "$d"
    echo

    if [[ -f "$d/report.md" ]]; then
        cat "$d/report.md"
    else
        echo "P4 aún no termina."
        echo
        echo "Batches:"
        tail -n 12 "$d/batches.tsv" | column -t -s $'\t'
    fi
}

case "${1:-run}" in
    run)
        run_all
        ;;
    dashboard)
        dashboard
        ;;
    slots)
        slots_view
        ;;
    watch)
        watch -n 1 "$SELF slots"
        ;;
    results)
        results_view
        ;;
    restore)
        restore_baseline
        ;;
    *)
        echo "Uso:"
        echo "  $SELF run"
        echo "  $SELF dashboard"
        echo "  $SELF slots"
        echo "  $SELF watch"
        echo "  $SELF results"
        echo "  $SELF restore"
        exit 1
        ;;
esac
