25. 评估流水线工程：四层接入 CI/CD

# .github/workflows/eval-pr.yml —— L1:PR 快速回归
name: PR Eval
on:
  pull_request:
    paths:
      - "prompts/**"     # prompt 改动必然影响分数,必须跑
      - "src/**"
      - "datasets/**"    # 数据集变更也要重跑,防"卷子变了"

jobs:
  eval:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
      - run: npm ci --ignore-scripts
      # 50 题 + 阈值 0.80:门禁粒度是 2%(一题一档),阈值必须校准后再启用阻断
      - name: Quick regression
        run: npx tsx mini-eval.ts datasets/pr-smoke.jsonl --threshold 0.80 --concurrency 8
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      - name: Comment result on PR
        if: always()
        run: node scripts/report-to-pr.js reports/last-run.json

# .github/workflows/eval-nightly.yml —— L2:夜间全量(趋势警报)
name: Nightly Eval
on:
  schedule:
    - cron: "0 19 * * *"   # UTC 19:00 = 北京时间凌晨 3 点,避开业务高峰
  workflow_dispatch:         # 支持手动触发

jobs:
  eval:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with: { node-version: 22 }
      - run: npm ci --ignore-scripts
      - name: Full eval (500 items)
        run: npx tsx mini-eval.ts datasets/nightly.jsonl --threshold 0.75 --concurrency 8
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      - name: Regression check vs baseline
        run: node scripts/check-regression.mjs reports/last-run.json reports/baseline.json
      - name: Upload reports
        uses: actions/upload-artifact@v4
        with: { name: eval-reports, path: reports/ }

if (当前 pass 率 < 基线 pass 率 - 0.02) alert();

// scripts/check-regression.mjs —— L2 回归检测(零依赖)
// 运行: node scripts/check-regression.mjs
function wilson(correct, n, z = 1.96) {
  if (n <= 0) throw new Error("n must be > 0");
  const p = correct / n, z2 = z * z;
  const d = 1 + z2 / n;
  const ctr = (p + z2 / (2 * n)) / d;
  const h = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / d;
  return [Math.max(0, ctr - h), Math.min(1, ctr + h)];
}
function erf(x) { const a1=0.254829592,a2=-0.284496736,a3=1.421413741,
  a4=-1.453152027,a5=1.061405429,p=0.3275911;const s=x<0?-1:1;x=Math.abs(x);
  const t=1/(1+p*x);const y=1-((((a5*t+a4)*t+a3)*t+a2)*t+a1)*t*Math.exp(-x*x);return s*y }
function normalCdf(z) { return 0.5 * (1 + erf(z / Math.SQRT2)); }

// 双比例 z 检验:合并比例算标准误
function twoProportionZ(cur, base) {
  const p1 = cur.pass / cur.total, p2 = base.pass / base.total;
  const pool = (cur.pass + base.pass) / (cur.total + base.total);
  const se = Math.sqrt(pool * (1 - pool) * (1 / cur.total + 1 / base.total));
  const z = (p1 - p2) / se;
  return { z: +z.toFixed(4), pValue: +(2 * (1 - normalCdf(Math.abs(z)))).toFixed(4) };
}

const MIN_DELTA = 0.03; // 业务最小可感知差值,与业务方约定后写入配置
export function detectRegression(current, baseline) {
  const curRate = current.pass / current.total, baseRate = baseline.pass / baseline.total;
  const [curLo, curHi] = wilson(current.pass, current.total);
  const [baseLo, baseHi] = wilson(baseline.pass, baseline.total);
  const { z, pValue } = twoProportionZ(current, baseline);
  const deltaBreached = curRate < baseRate - MIN_DELTA; // 业务上真的掉了
  const noiseExcluded = pValue < 0.05;                  // 统计上排除抽样噪声
  return {
    curRate: +curRate.toFixed(3), baseRate: +baseRate.toFixed(3),
    curCI: [curLo.toFixed(3), curHi.toFixed(3)], baseCI: [baseLo.toFixed(3), baseHi.toFixed(3)],
    deltaBreached, noiseExcluded, z, pValue,
    regressed: deltaBreached && noiseExcluded,
  };
}

// 案例 A: 50 题快集上 86% -> 80%——看起来吓人,统计上是噪声
const A = detectRegression({ pass: 40, total: 50 }, { pass: 43, total: 50 });
// 案例 B: 500 题全量上 86% -> 80%——同样的降幅,这次是真回归
const B = detectRegression({ pass: 400, total: 500 }, { pass: 430, total: 500 });
console.log("A(n=50):", JSON.stringify(A));
console.log("B(n=500):", JSON.stringify(B));
// 期望输出:
// A(n=50): {"curRate":0.8,"baseRate":0.86,"curCI":["0.670","0.888"],"baseCI":["0.738","0.930"],"deltaBreached":true,"noiseExcluded":false,"z":-0.7987,"pValue":0.4245,"regressed":false}
// B(n=500): {"curRate":0.8,"baseRate":0.86,"curCI":["0.763","0.833"],"baseCI":["0.827","0.888"],"deltaBreached":true,"noiseExcluded":true,"z":-2.5256,"pValue":0.0116,"regressed":true}

// scripts/alert-debouncer.mjs —— 告警去抖状态机(零依赖)
// 运行: node scripts/alert-debouncer.mjs
class AlertDebouncer {
  constructor(needConsecutive = 2, cooldownMs = 24 * 3600 * 1000) {
    this.need = needConsecutive; this.cooldown = cooldownMs;
    this.streak = 0; this.lastSentAt = 0;
  }
  shouldAlert(breached, now) {
    if (!breached) { this.streak = 0; return false; }        // 恢复正常,清零
    this.streak++;
    if (this.streak < this.need) return false;               // 未连续达标,不报
    if (now - this.lastSentAt < this.cooldown) return false; // 冷却期内,不重复报
    this.lastSentAt = now; this.streak = 0;
    return true;
  }
}

const HOUR = 3600 * 1000;
const d = new AlertDebouncer(2, 24 * HOUR);
let now = 0;
const breaches = [true, true, true, true, false, true, true];
breaches.forEach((b, i) => {
  now += 12 * HOUR; // 每 12 小时一个 L2 周期
  console.log(`周期${i + 1}(+12h) breached=${b} -> 告警=${d.shouldAlert(b, now)}`);
});
// 期望输出:
// 周期1(+12h) breached=true -> 告警=false   (连续 1 次,未达标)
// 周期2(+12h) breached=true -> 告警=true    (连续 2 次,触发)
// 周期3(+12h) breached=true -> 告警=false   (冷却期内)
// 周期4(+12h) breached=true -> 告警=true    (距上次 24h,再报一次提醒)
// 周期5(+12h) breached=false -> 告警=false  (恢复,清零)
// 周期6(+12h) breached=true -> 告警=false   (新一轮连续 1 次)
// 周期7(+12h) breached=true -> 告警=true    (连续 2 次,触发)

// scripts/notify-slack.mjs —— 告警通知(需要 SLACK_WEBHOOK 环境变量)
export async function notifySlack(report: {
  regressed: boolean; curRate: number; baseRate: number; z: number; pValue: number;
  worstCategory: { name: string; drop: number };
  failingItems: Array<{ id: string; category: string; input: string }>;
  runUrl: string;
}) {
  const lines = [
    `${report.regressed ? ":rotating_light: *夜间评估回归*" : ":white_check_mark: 夜间评估正常"}`,
    `pass 率: ${(report.curRate * 100).toFixed(1)}% (基线 ${(report.baseRate * 100).toFixed(1)}%)`,
    `统计: z=${report.z}, p=${report.pValue} (双比例 z 检验)`,
    `最差类别: ${report.worstCategory.name} 下滑 ${(report.worstCategory.drop * 100).toFixed(1)}pp`,
    `失败样本 Top3: ${report.failingItems.map((f) => `[${f.category}] ${f.input.slice(0, 30)}`).join(" | ")}`,
    `完整报告: ${report.runUrl}`,
  ];
  await fetch(process.env.SLACK_WEBHOOK!, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ text: lines.join("\n") }),
  });
}

# .github/workflows/eval-release.yml —— L3:发版安全集
name: Release Eval
on:
  push:
    tags: ["v*"]            # 发版 tag 触发

jobs:
  eval:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with: { node-version: 22 }
      - run: npm ci --ignore-scripts
      # 合规红线 0 容忍:任何一条失败都不允许发版
      - name: Safety gate (zero tolerance)
        run: npx tsx mini-eval.ts datasets/release-safety.jsonl --threshold 1.0 --concurrency 8
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      # 高频场景集:阈值语义是门禁,但不是 0 容忍
      - name: Core scenarios
        run: npx tsx mini-eval.ts datasets/release-core.jsonl --threshold 0.85 --concurrency 8
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

// scripts/backflow.mjs —— 失败样本回流(追加写入回流池,零依赖)
// 运行: node scripts/backflow.mjs reports/last-run.json _pool/reflow.jsonl
import { appendFileSync, readFileSync } from "node:fs";

const [,, reportPath, poolPath] = process.argv;
const run = JSON.parse(readFileSync(reportPath, "utf8"));
const failing = run.results.filter((r: any) =>
  r.scores.some((s: any) => s.value < run.threshold)
);
for (const f of failing) {
  appendFileSync(poolPath, JSON.stringify({
    kind: "ci-backflow",            // 与线上点踩回流区分来源
    itemId: f.id,
    input: f.input,                  // 冗余快照,防数据集演化后对不上
    output: f.output,
    scores: f.scores,
    capturedAt: new Date().toISOString(),
    runId: run.runId,
  }) + "\n");
}
console.log(`backflow: ${failing.length} items -> ${poolPath}`);
// 期望输出(以一次 3 条失败的 run 为例):
// backflow: 3 items -> _pool/reflow.jsonl