#!/usr/bin/env python3 """Render the local EP1 report from authoritative snapshots and supplementary checks. Never changes snapshots, runs, scores or credentials. Output works over file://. """ import hashlib, json, pathlib, runpy ROOT=pathlib.Path(__file__).resolve().parents[1] TEMPLATES=ROOT/'site/templates' def main(): files=[ROOT/'data/experiments/exp-20260915-ep1-f005-docker.json',ROOT/'data/experiments/exp-20260918-ep1-f005-glm-probe1200.json'] snapshots=[json.loads(p.read_text()) for p in files] evidence=json.loads((ROOT/'site/evidence/ep1-reference/comparison.json').read_text()) cases={c['id']:c for c in evidence['runs']} rows=[] for snap in snapshots: for raw in snap['runs']: if raw['task']!='ep1-f005':continue u=raw.get('wire_backfill') or raw['official_usage'] net=u.get('net_input',u.get('input_tokens')); cache=u.get('cache_read',u.get('cache_read_tokens')) row={ 'id':raw['run_id'],'model':raw['model'],'attempt':raw['attempt'],'timestamp':raw['ts'], 'helper':cases[raw['run_id']]['helper'],'sourceSha256':cases[raw['run_id']]['sha256'], 'budget':raw['params']['timeout_s'],'seconds':raw['wall_s'],'score':raw['score'],'exit':raw['agent_exit'], 'calls':u['api_calls'],'net':net,'cache':cache,'output':u.get('output',u.get('output_tokens')), 'usageSource':'wire' if raw.get('wire_backfill') else 'official', 'testModified':raw['score_diag'].get('test_modified_by_model'), 'checks':cases[raw['run_id']]['checks'],'mechanism':cases[raw['run_id']]['mechanism'], 'diff':(ROOT/'site/evidence/ep1-reference'/f"{raw['run_id']}.diff").read_text(), } assert all(row[k] is not None for k in ['net','cache','calls','output']) rows.append(row) rows.sort(key=lambda r:(r['budget'],r['model'],r['attempt'])) main_runs=[r for r in rows if r['budget']==420] passed=[r for r in rows if r['score']==1] main_models=set(r['model'] for r in main_runs) summary={ 'mainRuns':len(main_runs),'mainModels':len(main_models),'mainPassed':sum(r['score']==1 for r in main_runs), 'mainTimeouts':sum(r['exit']==124 for r in main_runs), 'mainModelsBothPassed':sum(all(r['score']==1 for r in main_runs if r['model']==m) for m in main_models), 'allTaskRuns':len(rows),'passedRuns':len(passed), 'passedCopyPass':sum(r['checks']['api_key_copy']['pass'] for r in passed), 'checksRequests':sum(len(c['checks']) for c in cases.values()), } assert len(rows)==20 and len(main_runs)==18 and len(main_models)==9 assert summary['mainPassed']==14 and summary['passedCopyPass']==0 answer_review=json.loads((ROOT/'site/evidence/ep1-reference/answer-review.json').read_text()) assert {r['id'] for r in answer_review['runs']}=={r['id'] for r in rows} assert sum(r['status']=='final' for r in answer_review['runs'])==16 for item in answer_review['runs']+answer_review['highlights']: source=ROOT/item['source'] assert hashlib.sha256(source.read_bytes()).hexdigest()==item['sourceSha256'], str(source) body=json.loads(source.read_text())['body'] visible='' for line in body.splitlines(): if not line.startswith('data: '):continue try: chunk=json.loads(line[6:]) except json.JSONDecodeError:continue for choice in chunk.get('choices',[]): visible+=choice.get('delta',choice.get('message',{})).get('content') or '' assert item.get('quote',item.get('text')) in visible, str(source) answer_corpus=json.loads((ROOT/'site/evidence/ep1-reference/answer-corpus.json').read_text()) answer_rubric=json.loads((ROOT/'site/evidence/ep1-reference/answer-rubric.json').read_text()) run_ids={r['id'] for r in rows} assert {r['id'] for r in answer_corpus['runs']}==run_ids assert {r['id'] for r in answer_rubric['runs']}==run_ids parse_reply=runpy.run_path(str(ROOT/'scripts/extract-ep1-answers.py'))['visible_reply'] for run in answer_corpus['runs']: for message in run['messages']: source=ROOT/message['source'] assert hashlib.sha256(source.read_bytes()).hexdigest()==message['sourceSha256'] text,tools=parse_reply(source) assert text==message['text'] and tools==message['hasToolCalls'] for item in answer_rubric['runs']: messages=next(r['messages'] for r in answer_corpus['runs'] if r['id']==item['id']) assert item['reviewedMessages']==len(messages) assert all(e in messages for e in item['evidence']) # 分层排名：应要求补充，只用主实验（420s 同预算）聚合，派生自快照 + 补测 + 审阅档位。 # 延时探针是另一种预算条件，混进来名次就不可比，故排除在外。 LOCATE={'explained':3,'proposed':2,'scoped_out':1,'not_observed':0} LOCATE_LABEL={3:'解释复制风险',2:'提出排查',1:'检查后限定范围',0:'未见明确表述'} rubric_by={r['id']:r for r in answer_rubric['runs']} by_model={} for r in main_runs: m=by_model.setdefault(r['model'],{'model':r['model'],'runs':[],'seconds':[],'locate':0,'mechanisms':set()}) m['runs'].append(r['id']); m['seconds'].append(r['seconds']) m['locate']=max(m['locate'],LOCATE[rubric_by[r['id']]['copyRisk']]) m['mechanisms'].add(r['mechanism']) ranking=[] for m in by_model.values(): rr=[r for r in main_runs if r['model']==m['model']] assert len(rr)==2, m['model'] ranking.append({ 'model':m['model'],'runs':m['runs'], 'passed':sum(r['score'] for r in rr), 'seconds':round(sum(m['seconds'])/len(m['seconds']),1), 'calls':round(sum(r['calls'] for r in rr)/2,1), 'locate':m['locate'],'locateLabel':LOCATE_LABEL[m['locate']], 'copyPass':sum(r['checks']['api_key_copy']['pass'] and r['checks']['bearer_copy']['pass'] for r in rr), 'mechanisms':sorted(m['mechanisms']), }) # 排序规则写死在这里，页面上原样公示，读者可据此自行重排。 ranking.sort(key=lambda r:(-r['passed'],-r['locate'],r['seconds'])) for i,r in enumerate(ranking,1): r['rank']=i reference_checks=cases['upstream-reference']['checks'] # 运行日期范围：对应 OpenRouter 排行榜把数据新鲜度写在标题下方的做法，同样由脚本从快照取。 run_days=sorted(r['timestamp'][:10] for r in main_runs) ranking_meta={ 'rule':'先按原题通过次数降序，再按复制边界表述档位降序，最后按两次耗时均值升序。', 'budget':420,'runsPerModel':2,'models':len(ranking), 'runs':len(main_runs),'firstDay':run_days[0],'lastDay':run_days[-1], 'locateSource':'单遍 AI 辅助文本审阅，无独立第二审阅者', 'byTime':[r['model'] for r in sorted([r for r in ranking if r['passed']==2],key=lambda r:r['seconds'])], 'copyWinners':sum(r['copyPass']>0 for r in ranking), 'referenceAllPass':all(c['pass'] for c in reference_checks.values()), 'locateLevels':LOCATE_LABEL, } assert ranking_meta['copyWinners']==0 and ranking_meta['referenceAllPass'] assert len(ranking)==9 and sum(r['passed'] for r in ranking)==summary['mainPassed'] # 首屏数据基准条：对应 OpenRouter 把数据新鲜度写在标题下的做法（openrouter.ai/rankings，2026-09-21 读取）。 # 全部字段由脚本从快照重算，不接受手填。 all_days=sorted(r['timestamp'][:10] for r in rows) data_base={ 'runs':len(rows),'models':len(main_models),'runsPerModel':2, 'budgets':sorted({r['budget'] for r in rows}), 'firstDay':all_days[0],'lastDay':all_days[-1], 'snapshots':[{'name':f.name,'label':lbl,'sha256':hashlib.sha256(f.read_bytes()).hexdigest()[:12]} for f,lbl in zip(files,('主实验快照','探针快照'))], } assert data_base['runs']==20 and data_base['models']==9 and data_base['budgets']==[420,1200] data={ 'answerReview':answer_review,'answerCorpus':answer_corpus,'answerRubric':answer_rubric, 'runs':rows,'summary':summary,'ranking':ranking,'rankingMeta':ranking_meta,'fixture':snapshots[0]['fixture'], 'dataBase':data_base, 'reference':cases['upstream-reference'],'baseline':cases['baseline'], 'checkDate':evidence['generated_at'],'sdk':evidence['anthropic_version'], 'findings':json.loads((TEMPLATES/'ep1-findings.json').read_text()), 'sourceHashes':{str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in files}, } payload=json.dumps(data,ensure_ascii=False,separators=(',',':')).replace('<','\\u003c').replace('>','\\u003e').replace('&','\\u0026') for template,filename in [('ep1-report.html','ep1-douban.html'),('ep1-evidence.html','ep1-evidence.html')]: out=(TEMPLATES/template).read_text().replace('/* REPORT_TOKENS */',(TEMPLATES/'ep1-tokens.css').read_text()).replace('/* REPORT_DATA */',payload) (ROOT/'site/drafts'/filename).write_text(out) print(json.dumps(summary,ensure_ascii=False)) if __name__=='__main__':main()