28. 案例研究（二）：代码 Agent 的三层评估落地

// code-agent/types.ts —— 建议生命周期事件契约（无需联网）
// 每一次"插件给出建议"都会产生一条 suggestion 事件，之后跟随一条结果事件

export interface SuggestionEvent {
  id: string;
  ts: number;
  userId: string;             // 脱敏后的设备标识
  kind: "inline" | "function" | "fix"; // 补全 / 函数生成 / Bug 修复
  language: "ts" | "tsx" | "js" | "python";
  model: string;              // 被测模型版本
  promptVersion: string;      // 提示词版本——三层归因的关键字段
  latencyMs: number;
  shown: boolean;             // 是否真正展示给用户（防止把未展示算进分母）
}

export type SuggestionOutcome =
  | { id: string; action: "accepted"; editedLines: number; finalCode: string }
  | { id: string; action: "rejected" }
  | { id: string; action: "ignored"; dwellMs: number }  // 展示后未交互
  | { id: string; action: "expired" };                  // 用户已输入，建议作废

// code-agent/run-suite.ts —— 沙箱执行生成的 TS 代码（本地运行；测试样本需自备）
// 运行：npx tsx code-agent/run-suite.ts data/ts-tasks.jsonl
import { spawnSync } from "node:child_process";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { readFileSync } from "node:fs";

interface TsTask { id: string; prompt: string; test: string; category: string; }

export function runOne(task: TsTask, completion: string): { pass: boolean; reason: string } {
  const dir = mkdtempSync(join(tmpdir(), "code-eval-"));
  try {
    // 生成的代码 + 测试写进独立临时目录：互相不可见，跑完即删
    writeFileSync(join(dir, "solution.ts"), `${task.prompt}\n${completion}\n${task.test}\n`);
    const r = spawnSync("npx", ["tsx", join(dir, "solution.ts")], {
      cwd: dir,
      timeout: 10_000,          // 必须有超时：死循环是模型的高频失败模式
      stdio: "pipe",
      env: { ...process.env, NODE_OPTIONS: "--no-experimental-strip-types" },
    });
    // 退出码 0 且 stderr 无未捕获异常 → 通过；其余一律按失败并保留原因
    const clean = r.status === 0 && !/Unhandled error|AssertionError/.test(r.stderr.toString());
    return { pass: clean, reason: clean ? "ok" : r.stderr.toString().slice(0, 300) };
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
}

const tasks: TsTask[] = readFileSync(process.argv[2] ?? "data/ts-tasks.jsonl", "utf-8")
  .trim().split("\n").map(JSON.parse);
let pass = 0;
for (const t of tasks) {
  const completion = await solve(t);        // solve：调用被测模型，见第 20 章 17.3
  const r = runOne(t, completion);
  if (r.pass) pass++;
  else console.error(`FAIL ${t.id} [${t.category}] ${r.reason.split("\n")[0]}`);
}
console.log(`pass_rate=${(pass / tasks.length).toFixed(3)} n=${tasks.length}`);
// 期望输出示例：pass_rate=0.875 n=200

// code-agent/biz-metrics.ts —— 从事件日志聚合业务层指标（本地运行，无需联网）
// 运行：npx tsx code-agent/biz-metrics.ts data/events.jsonl
import { readFileSync } from "node:fs";

type Event = import("./types.ts").SuggestionEvent | import("./types.ts").SuggestionOutcome;

const lines = readFileSync(process.argv[2] ?? "data/events.jsonl", "utf-8").trim().split("\n");
const events: Event[] = lines.map(JSON.parse);

const shown = new Map(events.filter(e => "kind" in e && e.shown).map(e => [e.id, e]));
const outcomes = events.filter(e => "action" in e && shown.has(e.id));

const accepted = outcomes.filter(o => o.action === "accepted");
const rejected = outcomes.filter(o => o.action === "rejected");
const ignored  = outcomes.filter(o => o.action === "ignored");

const acceptance = accepted.length / Math.max(accepted.length + rejected.length, 1);
const perfect    = accepted.filter(o => o.editedLines === 0).length / Math.max(accepted.length, 1);
const ignoreRate = ignored.length / shown.size;

console.log({
  acceptanceRate: acceptance.toFixed(3),      // 展示接受率
  perfectAcceptRate: perfect.toFixed(3),      // 零修改直接接受
  ignoreRate: ignoreRate.toFixed(3),          // 展示后无交互（单列观察，不进主指标）
  n: accepted.length + rejected.length,
});
// 期望输出示例：{ acceptanceRate: '0.371', perfectAcceptRate: '0.588', ignoreRate: '0.212', n: 8620 }

1. 筛选：近 12 个月已合并的 bugfix PR，过滤条件——
   · 带有可复现的测试改动（test patch 存在）
   · 改动 ≤ 5 个源文件（排除大规模重构）
   · 模块分布覆盖 P0 路径（结算、登录、权限），不是只挑好修的
2. 固化：每个 Issue 记录 base_commit / issue 正文 / golden patch / test patch
3. 脱敏：业务字段改名、删除内网地址——内测集会进 CI 日志
4. 版本：swe-cases v1.0 锁定（30 例），改动永远新建版本

// code-agent/swe-case.ts —— 内测集单例执行（需本机 git 与 node；Agent 调用需联网）
// 运行：npx tsx code-agent/swe-case.ts swe-cases/issue-0142.json
import { spawnSync } from "node:child_process";
import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

interface SweCase {
  id: string; repoUrl: string; baseCommit: string;
  issueText: string; goldenPatch: string; testPatch: string; testCmd: string;
}

function sh(cmd: string, cwd: string, timeoutMs = 300_000) {
  return spawnSync("bash", ["-c", cmd], { cwd, timeout: timeoutMs, stdio: "pipe" });
}

export async function runSweCase(c: SweCase, producePatch: (issue: string, cwd: string) => Promise<string>) {
  const dir = mkdtempSync(join(tmpdir(), "swe-"));
  try {
    // 1. 固定到 Issue 发生前的提交，保证起点一致
    sh(`git clone --quiet ${c.repoUrl} . && git checkout --quiet ${c.baseCommit}`, dir);

    // 2. Agent 只拿到 Issue 正文与工作目录，产出 unified diff
    const patch = await producePatch(c.issueText, dir);
    writeFileSync(join(dir, "model.patch"), patch);
    if (sh("git apply --whitespace=nowarn model.patch", dir).status !== 0)
      return { id: c.id, pass: false, reason: "patch 不适用" };

    // 3. 测试补丁单独打：Agent 看不到测试内容，防止"对着测试写代码"
    writeFileSync(join(dir, "test.patch"), c.testPatch);
    sh("git apply test.patch", dir);

    // 4. 唯一裁判是仓库自己的测试
    const r = sh(c.testCmd, dir);
    return { id: c.id, pass: r.status === 0, reason: r.status === 0 ? "ok" : "tests failed" };
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
}

// 自检入口：golden patch 必须满分（放进 CI，断言失败即整个内测集不可信）
const c: SweCase = JSON.parse(readFileSync(process.argv[2]!, "utf-8"));
const verdict = await runSweCase(c, async () => c.goldenPatch);
console.log(verdict.pass ? "harness 自检通过" : "harness 有缺陷：golden patch 未通过");
process.exitCode = verdict.pass ? 0 : 1;

// code-agent/agreement.ts —— 评分员一致性（本地运行，无需联网）
// 运行：npx tsx code-agent/agreement.ts data/weekly-ratings.json
import { readFileSync } from "node:fs";

// ratings: [{ id, rater: "a"|"b"|"c", scores: { correctness, style, maintainability, security } }]
const ratings: Array<{ id: string; rater: string; scores: Record<string, number> }> =
  JSON.parse(readFileSync(process.argv[2]!, "utf-8"));

export function cohensKappa(a: number[], b: number[]): number {
  const cats = [...new Set([...a, ...b])];
  const n = a.length;
  const po = a.filter((v, i) => v === b[i]).length / n;                    // 观察一致率
  const pe = cats.reduce((s, c) =>
    s + (a.filter(v => v === c).length / n) * (b.filter(v => v === c).length / n), 0);
  return (po - pe) / (1 - pe);                                             // 校正随机一致
}

const ids = [...new Set(ratings.map(r => r.id))];
for (const dim of ["correctness", "style", "maintainability", "security"] as const) {
  const pair = ids.filter(id => ratings.filter(r => r.id === id).length >= 2)
    .map(id => ratings.filter(r => r.id === id).slice(0, 2).map(r => r.scores[dim]));
  const [a, b] = [pair.map(p => p[0]), pair.map(p => p[1])];
  console.log(`${dim}: kappa=${cohensKappa(a, b).toFixed(2)} n=${a.length}`);
}
// 期望输出示例：correctness: kappa=0.74 n=50  （低于 0.4 的维度先修 rubric）