#!/usr/bin/env python3
"""Summarize completed and partial shared-benchmark evidence, without promotion."""
import argparse
import json
import statistics
from pathlib import Path


def median(values):
    data = [v for v in values if isinstance(v, (int, float))]
    return round(statistics.median(data), 3) if data else None


def summarize(path):
    groups = {}
    for line in path.read_text().splitlines():
        try:
            row = json.loads(line)
        except json.JSONDecodeError:
            continue  # May be read while the writer is appending a row.
        key = (row['length_budget'], row['concurrency'])
        groups.setdefault(key, []).append(row)
    summary_file = path.parent / 'summary.json'
    metadata = json.loads(summary_file.read_text()) if summary_file.exists() else {}
    gpu_file = path.parent / 'gpu.json'
    memory = []
    if gpu_file.exists():
        for sample in json.loads(gpu_file.read_text()):
            try:
                memory.append(float(sample['values'].split(',')[0]))
            except (KeyError, ValueError):
                pass
    rows = []
    for (context, concurrency), batches in sorted(groups.items()):
        results = [r for b in batches for r in b['results']]
        tokens = [r.get('usage', {}).get('prompt_tokens') for r in results]
        tokens = [v for v in tokens if isinstance(v, int)]
        rows.append({
            'context_budget': context, 'http_concurrency': concurrency,
            'batches': len(batches), 'requests': len(results),
            'http_success': sum(bool(r.get('ok')) for r in results),
            'output_nonempty': sum(bool(r.get('content', '').strip()) for r in results),
            'retrieval_evaluated': sum(r.get('retrieval_pass') is not None for r in results),
            'retrieval_pass': sum(bool(r.get('retrieval_pass')) for r in results),
            'cross_request_marker_evaluated': sum('foreign_request_markers' in r for r in results),
            'cross_request_marker_failures': sum(bool(r.get('foreign_request_markers')) for r in results),
            'sse_terminal_recorded': sum('sse_done' in r for r in results),
            'prompt_tokens_min': min(tokens) if tokens else None,
            'prompt_tokens_max': max(tokens) if tokens else None,
            'ttft_median_seconds': median(r.get('ttft_seconds') for r in results),
            'decode_median_tok_s_estimate': median(r.get('decode_tok_s_estimate') for r in results),
            'e2e_median_tok_s': median(r.get('e2e_tok_s') for r in results),
            'aggregate_median_tok_s': median(b.get('aggregate_tok_s') for b in batches),
            'errors': sorted({r.get('error', 'unknown error') for r in results if not r.get('ok')}),
        })
    return {
        'source': str(path), 'execution_ended': bool(metadata),
        'completed_grid': metadata.get('completed'), 'reported_all_pass': metadata.get('all_pass'),
        'arguments': metadata.get('arguments', {}),
        'task': metadata.get('arguments', {}).get('task', 'retrieval'),
        'wall_seconds': metadata.get('wall_seconds'),
        'gpu_reserved_peak_mib': max(memory) if memory else None, 'rows': rows,
        'caveats': ['HTTP concurrency does not itself prove simultaneous GPU residency.',
                    'SSE-derived decode rate is an estimate; chunking can batch output tokens.',
                    'GPU memory is sampled total including reserved cache, not weight size.'],
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('root', type=Path)
    parser.add_argument('--out', type=Path)
    args = parser.parse_args()
    reports = []
    for path in sorted(args.root.rglob('results.jsonl')):
        try:
            reports.append(summarize(path))
        except (KeyError, ValueError) as exc:
            reports.append({'source': str(path), 'error': str(exc)})
    text = json.dumps(reports, ensure_ascii=False, indent=2) + '\n'
    if args.out:
        args.out.write_text(text)
    else:
        print(text, end='')


if __name__ == '__main__':
    main()
