20. 用 Node.js 自建 Mini Evaluator：从 30 行到四层评估流水线

数据集(JSONL) → 被测物(solve 函数) → 判官(规则/LLM) → 汇总(指标+报告)
      ↑                                                    │
      └────────────── 回归：改了 prompt/模型后重跑同一数据集 ←┘

// mini-eval-30.ts —— 运行: OPENAI_API_KEY=sk-xxx npx tsx mini-eval-30.ts
// 需联网与 API Key；50 题约产生几美分费用（以 gpt-4o-mini 计价）
import { readFileSync } from "node:fs";
import OpenAI from "openai";

const openai = new OpenAI();

type Task = { input: string; expected: string };
const tasks: Task[] = readFileSync("data.jsonl", "utf-8")
  .trim().split("\n").map((l) => JSON.parse(l));

let correct = 0;
for (const t of tasks) {
  const r = await openai.chat.completions.create({
    model: "gpt-4o-mini", // 占位模型名，替换为你账号可用的模型
    messages: [{ role: "user", content: t.input }],
  });
  if (r.choices[0].message.content?.trim() === t.expected) correct++;
}
console.log(`Accuracy: ${((correct / tasks.length) * 100).toFixed(1)}%`);

// mini-eval-100.ts —— 支持：有界并发 / 缓存 / 指数退避重试 / 分类报告 / 结果落盘
// 运行: OPENAI_API_KEY=sk-xxx npx tsx mini-eval-100.ts
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
import OpenAI from "openai";
import pLimit from "p-limit";
import { LRUCache } from "lru-cache";

const openai = new OpenAI();
const MODEL = "gpt-4o-mini";

// 1) 缓存：prompt 全文作 key，命中即零成本（重复跑评估最省钱的手段）
const cache = new LRUCache<string, string>({ max: 10000, ttl: 24 * 3600 * 1000 });

// 2) 重试：指数退避 1s/2s/4s，上限 3 次——429 与 5xx 都靠它扛
async function callWithRetry(prompt: string, retries = 3): Promise<string> {
  const hit = cache.get(prompt);
  if (hit !== undefined) return hit;
  for (let i = 0; i < retries; i++) {
    try {
      const r = await openai.chat.completions.create({
        model: MODEL,
        messages: [{ role: "user", content: prompt }],
        temperature: 0, // 温度 0：同一 prompt 复跑结果稳定，缓存才有意义
      });
      const out = r.choices[0].message.content ?? "";
      cache.set(prompt, out);
      return out;
    } catch (e) {
      if (i === retries - 1) throw e;
      await new Promise((r) => setTimeout(r, 1000 * (i + 1)));
    }
  }
  throw new Error("unreachable");
}

// 3) 判官：严格相等（业务上可换成 contains / 数值抽取 / LLM 判官）
interface Task { id: string; input: string; expected: string; category?: string; }
function exactMatch(output: string, expected: string): boolean {
  return output.trim() === expected.trim();
}

const tasks: Task[] = readFileSync("data.jsonl", "utf-8")
  .trim().split("\n").map((l, i) => ({ id: `ex-${i}`, ...JSON.parse(l) }));

// 4) 有界并发：pLimit(10) —— 同时最多 10 个在途请求，不触发限流
const limit = pLimit(10);
const results = await Promise.all(
  tasks.map((t) =>
    limit(async () => {
      const output = await callWithRetry(t.input);
      return { ...t, output, correct: exactMatch(output, t.expected) };
    })
  )
);

// 5) 报告：总体 + 分桶（分桶才能看见"高频场景还行、长尾全崩"这种回归）
mkdirSync("reports", { recursive: true });
const total = results.length;
const correct = results.filter((r) => r.correct).length;
const byCategory: Record<string, { total: number; correct: number }> = {};
for (const r of results) {
  const cat = r.category ?? "default";
  byCategory[cat] ??= { total: 0, correct: 0 };
  byCategory[cat].total++;
  if (r.correct) byCategory[cat].correct++;
}
console.log(`Model: ${MODEL}, Total: ${total}, Acc: ${((correct / total) * 100).toFixed(1)}%`);
for (const [cat, s] of Object.entries(byCategory)) {
  console.log(`  ${cat}: ${((s.correct / s.total) * 100).toFixed(1)}% (${s.correct}/${s.total})`);
}
// 6) 落盘：原始结果留存，debug 与跨 run 对比的前提
writeFileSync("reports/last-run.jsonl", results.map((r) => JSON.stringify(r)).join("\n"));

// mini-eval.ts —— 依赖: npm i openai；Node >= 20
// 运行: OPENAI_API_KEY=sk-xxx npx tsx mini-eval.ts dataset.jsonl --threshold 0.8 --concurrency 4
// dataset.jsonl 每行: {"input":"...","expected":"..."}
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
import OpenAI from "openai";

// ---------- 类型：一次评估 run 的完整快照 ----------
type Example = { id: string; input: string; expected?: string };
type Score = { key: string; value: number; comment?: string };
type RowResult = { id: string; output: string; scores: Score[]; latencyMs: number };
type EvalRun = {
  runId: string; startedAt: string; dataset: string; itemCount: number;
  threshold: number; results: RowResult[];
  aggregates: Record<string, { value: number; ci95: [number, number] }>;
  passed: boolean;
};

// ---------- 判官协议：规则判官与 LLM 判官同一个签名，可插拔 ----------
type Judge = (ex: Example, output: string) => Promise<Score>;

const exactContains: Judge = async (ex, output) => ({
  key: "contains_expected",
  value: ex.expected && output.toLowerCase().includes(ex.expected.toLowerCase()) ? 1 : 0,
});

const llmJudge =
  (client: OpenAI, model = "gpt-4o-mini"): Judge =>
  async (ex, output) => {
    const r = await client.chat.completions.create({
      model,
      temperature: 0,
      response_format: { type: "json_object" }, // 结构化输出：分数 + 理由
      messages: [
        {
          role: "system",
          content:
            "你是评估判官。按 0~1 打分评估 ANSWER 是否正确回答 QUESTION。只输出 JSON: " +
            '{"score": number, "reason": string}',
        },
        { role: "user", content: `QUESTION: ${ex.input}\nANSWER: ${output}\nEXPECTED: ${ex.expected ?? "(无参考)"}` },
      ],
    });
    const parsed = JSON.parse(r.choices[0].message.content ?? "{}") as { score: number; reason?: string };
    return { key: "llm_judge", value: Math.min(1, Math.max(0, Number(parsed.score))), comment: parsed.reason };
  };

// ---------- 统计护栏：Wilson 95% 置信区间 ----------
function wilson(p: number, n: number, z = 1.96): [number, number] {
  if (n === 0) return [0, 0];
  const d = 1 + (z * z) / n;
  const c = (p + (z * z) / (2 * n)) / d;
  const h = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / d;
  return [c - h, c + h];
}

// ---------- 有界并发 map：固定 worker 池，不产生无限 Promise ----------
async function mapPool<T, R>(items: T[], limit: number, fn: (t: T) => Promise<R>): Promise<R[]> {
  const out: R[] = new Array(items.length);
  let i = 0;
  const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
    while (i < items.length) {
      const idx = i++; // 抢占式取号：每个 worker 从共享游标拿下一个任务
      out[idx] = await fn(items[idx]);
    }
  });
  await Promise.all(workers);
  return out;
}

// ---------- 参数解析：命名参数，避免位置索引陷阱 ----------
function parseArgs(argv: string[]) {
  const flag = (name: string) => {
    const i = argv.indexOf(name);
    return i >= 0 ? argv[i + 1] : undefined;
  };
  return {
    file: argv[0],
    threshold: Number(flag("--threshold") ?? 0.8),
    concurrency: Number(flag("--concurrency") ?? 4),
  };
}

// ---------- 主流程 ----------
async function main() {
  const { file, threshold, concurrency } = parseArgs(process.argv.slice(2));

  const examples: Example[] = readFileSync(file, "utf8")
    .split("\n").filter(Boolean)
    .map((line, i) => ({ id: `ex-${i}`, ...JSON.parse(line) }));

  const client = new OpenAI();
  // 被测物：换成你应用的真实入口即可（一个 (input) => Promise<string> 的函数）
  const solve = async (ex: Example) =>
    (await client.chat.completions.create({
      model: "gpt-4o-mini",
      messages: [{ role: "user", content: ex.input }],
    })).choices[0].message.content ?? "";

  const startedAt = new Date().toISOString();
  const results = await mapPool(examples, concurrency, async (ex) => {
    const t0 = Date.now();
    try {
      const output = await solve(ex);
      const scores = await Promise.all([exactContains(ex, output), llmJudge(client)(ex, output)]);
      return { id: ex.id, output, scores, latencyMs: Date.now() - t0 };
    } catch (e) {
      // 失败隔离：单条报错记为 0 分并留痕，不拖垮整个 run
      return {
        id: ex.id, output: "",
        scores: [{ key: "error", value: 0, comment: String(e).slice(0, 200) }],
        latencyMs: Date.now() - t0,
      };
    }
  });

  // 聚合：一条样本通过 = 所有判官都达到阈值（对应 DeepEval 的严格 pass 语义）
  const passRows = results.filter((r) => r.scores.every((s) => s.value >= threshold));
  const passRate = passRows.length / results.length;
  const [lo, hi] = wilson(passRate, results.length);

  const run: EvalRun = {
    runId: crypto.randomUUID(),
    startedAt,
    dataset: file,
    itemCount: examples.length,
    threshold,
    results,
    aggregates: {
      pass_rate: { value: passRate, ci95: [lo, hi] },
      avg_latency_ms: {
        value: results.reduce((s, r) => s + r.latencyMs, 0) / results.length,
        ci95: [0, 0],
      },
    },
    passed: passRate >= threshold,
  };

  // 报告 + 落盘（可复现：runId 对应一份完整快照）
  mkdirSync("reports", { recursive: true });
  writeFileSync(`reports/${run.runId}.json`, JSON.stringify(run, null, 2));
  console.log(`pass_rate=${passRate.toFixed(3)} ci95=[${lo.toFixed(3)},${hi.toFixed(3)}]`);
  console.log(`report=reports/${run.runId}.json`);

  // CI 门禁语义：失败让进程以非零码退出，CI 自动变红
  process.exitCode = run.passed ? 0 : 1;
}

main().catch((e) => { console.error(e); process.exitCode = 1; });

// judge-health.ts —— 判官一致率计算（自写，无框架依赖）
// gold: 人工定标集 [{ id, human: 0|1 }]；judgeScores: 判官对同批样本的打分 Map<id, 0|1>
function judgeAgreement(
  gold: { id: string; human: number }[],
  judge: Map<string, number>
) {
  const rows = gold.filter((g) => judge.has(g.id));
  const agree = rows.filter((g) => judge.get(g.id) === g.human).length;
  const n = rows.length;
  const p = agree / n;
  // Wilson 95% 置信区间：一致率也要报区间，不只报点估计
  const z = 1.96;
  const denom = 1 + (z * z) / n;
  const center = (p + (z * z) / (2 * n)) / denom;
  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denom;
  return { n, agree, p, ci95: [center - half, center + half] as [number, number] };
}

// 两段式判官：贴线样本才花贵模型的钱
const PASS_LINE = 0.8;
const MARGIN = 0.15;
async function tieredJudge(ex: Example, output: string) {
  const cheap = await llmJudge(cheapClient, "gpt-4o-mini")(ex, output);
  if (Math.abs(cheap.value - PASS_LINE) > MARGIN) return cheap; // 离线远：便宜判官说了算
  return llmJudge(strongClient, "gpt-4o")(ex, output);          // 贴线：升级复核
}

// langfuse-experiment.ts —— 官方 experiment runner 用法（整理注释）
// 运行: 需设置 LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY / LANGFUSE_HOST 与 OPENAI_API_KEY
import { OpenAI } from "openai";
import { NodeSDK } from "@opentelemetry/sdk-node";
import { LangfuseClient, ExperimentTask, ExperimentItem } from "@langfuse/client";
import { observeOpenAI } from "@langfuse/openai";
import { LangfuseSpanProcessor } from "@langfuse/otel";

const otelSdk = new NodeSDK({ spanProcessors: [new LangfuseSpanProcessor()] });
otelSdk.start();

const langfuse = new LangfuseClient();

// 数据集：本地数组或远端 Langfuse dataset
const localData: ExperimentItem[] = [
  { input: "What is the capital of France?", expectedOutput: "Paris" },
  { input: "What is the capital of Germany?", expectedOutput: "Berlin" },
];

// 被测任务：observeOpenAI 让每次调用自动成为 OTel span 上报
const myTask: ExperimentTask = async (item) => {
  const response = await observeOpenAI(new OpenAI()).chat.completions.create({
    model: "gpt-4o-mini",
    messages: [{ role: "user", content: item.input }],
  });
  return response;
};

// item 级判官：签名固定 { input, output, expectedOutput, metadata }
const accuracyEvaluator = async ({ output, expectedOutput }) => ({
  name: "accuracy",
  value: expectedOutput && output.toLowerCase().includes(expectedOutput.toLowerCase()) ? 1.0 : 0.0,
  comment: "substring match",
});

// run 级聚合判官：对整次 experiment 汇总
const averageAccuracy = async ({ itemResults }) => {
  const acc = itemResults.flatMap((r) => r.evaluations).filter((e) => e.name === "accuracy");
  if (!acc.length) return { name: "avg_accuracy", value: null };
  return { name: "avg_accuracy", value: acc.reduce((s, e) => s + e.value, 0) / acc.length };
};

const result = await langfuse.experiment.run({
  name: "Geography Quiz",
  data: localData,
  task: myTask,
  evaluators: [accuracyEvaluator],
  runEvaluators: [averageAccuracy],
  maxConcurrency: 5, // 有界并发，与 17.5 的 mapPool 同一思想
  metadata: { model: "gpt-4o-mini", version: "v1.2.0" },
});

console.log(await result.format());
await otelSdk.shutdown(); // serverless 环境必须显式关闭，否则丢 trace

# .github/workflows/pr-eval.yml —— L1：PR 快速回归（50 题硬门禁）
name: PR Eval
on:
  pull_request:

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
      - run: npm ci --ignore-scripts
      # threshold 与 L1 数据集规模匹配：50 题上阈值定太紧会天天误红（17.6.2）
      - run: npx tsx mini-eval.ts datasets/pr-smoke.jsonl --threshold 0.8 --concurrency 8
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

# .github/workflows/nightly-eval.yml —— L2：夜间全量（500 题，趋势警报）
name: Nightly Eval
on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
      - run: npm ci --ignore-scripts
      - run: npx tsx mini-eval.ts datasets/nightly.jsonl --threshold 0.75 --concurrency 8
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      - uses: actions/upload-artifact@v4
        with:
          name: eval-report
          path: reports/