5. LLM-as-Judge 工程化：从判官 prompt 到偏差控制

// minimal-judge.ts —— 最小判官：判断回答是否正面回应了问题
// 依赖：npm i openai
// 运行：OPENAI_API_KEY=sk-占位 npx tsx minimal-judge.ts （需联网，付费）
import OpenAI from "openai";
const client = new OpenAI();

export async function isRelevant(question: string, answer: string): Promise<boolean> {
  const r = await client.chat.completions.create({
    model: "gpt-4o-mini-2024-07-18", // pin 快照版本，不用漂浮别名（第 4 章可复现性）
    temperature: 0,                  // 判分要可复现：禁随机采样
    messages: [
      { role: "system", content: "你是评估判官。只回答 yes 或 no。" },
      { role: "user", content: `问题：${question}\n回答：${answer}\n这个回答是否正面回应了问题？` },
    ],
  });
  return (r.choices[0].message.content ?? "").trim().toLowerCase().startsWith("yes");
}

// score-judge.ts —— 单答案打分判官：rubric 五要素完整实现
// 依赖：npm i openai；Node >= 20
// 运行：OPENAI_API_KEY=sk-占位 npx tsx score-judge.ts （需联网，付费）
import OpenAI from "openai";
const client = new OpenAI();

// ---------- 要素 1-3：角色、判据锚点、判例，全部声明式 ----------
export type Rubric = {
  role: string;                        // 要素 1：角色——判官是谁、依据什么判
  criteria: {                          // 要素 2：判据——行为锚点，不是形容词
    name: string;
    weight: number;
    anchors: Record<string, string>;   // 档位 → 证据条件
  }[];
  examples?: {                         // 要素 3：判例（few-shot，含边界案例）
    question: string; answer: string; expected: Record<string, number>;
  }[];
  neutrality: string[];                // 反偏差声明（长度、格式中立，见 18.6.3）
  scale: [number, number];
};

export const RUBRIC_V1: Rubric = {
  role: "你是客服知识库问答的质量评估专家。你只依据<参考上下文>判分，不做上下文之外的补充推断。",
  criteria: [
    {
      name: "faithfulness", weight: 0.5,
      anchors: {
        "5": "每个论断都能在参考上下文中找到原句支撑",
        "3": "主体论断有支撑，但存在 1-2 处超出上下文的引申",
        "1": "出现上下文明确不支持的编造内容",
      },
    },
    {
      name: "relevancy", weight: 0.3,
      anchors: {
        "5": "直接回应提问的全部要点",
        "3": "只覆盖部分要点，或夹杂明显离题内容",
        "1": "答非所问",
      },
    },
    {
      name: "clarity", weight: 0.2,
      anchors: {
        "5": "结构清晰，一次读完即可理解",
        "1": "表述混乱，需要反复重读",
      },
    },
  ],
  examples: [{
    question: "退货期限是多久？",
    answer: "根据政策，签收后 30 天内可退货（参考上下文第 2 条）。",
    expected: { faithfulness: 5, relevancy: 5, clarity: 5 },
  }],
  neutrality: [
    "评分与回答长度无关：短而准确的回答不因太短扣分。",
    "评分与排版格式无关：不因使用或未使用 Markdown、列表、加粗而加减分。",
  ],
  scale: [1, 5],
};

// ---------- 要素 4：输出契约（JSON + 理由） ----------
export function buildPrompt(rubric: Rubric, question: string, answer: string, context: string): string {
  const dims = rubric.criteria
    .map(c => {
      const anchors = Object.entries(c.anchors).map(([s, cond]) => `  ${s} 分：${cond}`).join("\n");
      return `- ${c.name}（权重 ${c.weight}）\n${anchors}`;
    })
    .join("\n");
  const fields = rubric.criteria.map(c => `"${c.name}": <1-${rubric.scale[1]}>`).join(", ");
  const parts = [
    `# 角色\n${rubric.role}`,
    `# 评分判据（按档位锚点判分）\n${dims}`,
    `# 中立性约束\n${rubric.neutrality.map(s => `- ${s}`).join("\n")}`,
    `# 待评内容\n<问题>${question}</问题>\n<参考上下文>${context}</参考上下文>\n<回答>${answer}</回答>`,
    `# 输出契约\n只输出一个 JSON 对象：{"perDim":{${fields}},"reason":"逐判据给出一句话依据"}`,
  ];
  if (rubric.examples?.length) {
    const ex = rubric.examples
      .map((e, i) => `判例 ${i + 1}：问题「${e.question}」回答「${e.answer}」→ ${JSON.stringify(e.expected)}`)
      .join("\n");
    parts.splice(3, 0, `# 判例（few-shot）\n${ex}`);
  }
  return parts.join("\n\n");
}

export type ScoreResult =
  | { status: "ok"; total: number; perDim: Record<string, number>; reason: string }
  | { status: "parse_error"; raw: string };

export async function judgeOnce(
  model: string, rubric: Rubric, question: string, answer: string, context: string,
): Promise<ScoreResult> {
  const r = await client.chat.completions.create({
    model,                     // 必须传快照版本，如 gpt-4o-mini-2024-07-18
    temperature: 0,            // 判分要可复现
    response_format: { type: "json_object" },
    messages: [{ role: "user", content: buildPrompt(rubric, question, answer, context) }],
  });
  const raw = r.choices[0].message.content ?? "";
  try {
    const j = JSON.parse(raw) as { perDim: Record<string, number>; reason?: string };
    const perDim: Record<string, number> = {};
    let total = 0;
    for (const c of rubric.criteria) {
      const v = Math.min(rubric.scale[1], Math.max(rubric.scale[0], Number(j.perDim[c.name])));
      perDim[c.name] = v;
      total += v * c.weight;
    }
    return { status: "ok", total, perDim, reason: j.reason ?? "" };
  } catch {
    return { status: "parse_error", raw };   // 单独统计，不混入分数（见 18.8 陷阱 1）
  }
}

// ---------- 要素 5：阈值（业务决策，与 rubric 分开声明） ----------
export const GATE_V1 = { totalMin: 4.0, blocking: ["faithfulness"] as string[] };

export function passes(res: ScoreResult, gate: typeof GATE_V1): boolean | "invalid" {
  if (res.status !== "ok") return "invalid";
  const blocked = gate.blocking.some(d => (res.perDim[d] ?? 0) < 3); // 合取语义：一票否决
  return !blocked && res.total >= gate.totalMin;
}

// reference-judge.ts —— 有参考答案时的语义等价判官
// 运行：OPENAI_API_KEY=sk-占位 npx tsx reference-judge.ts （需联网，付费）
export const semanticEquivalent = async (params: {
  outputs: { sql: string };
  referenceOutputs?: { sql: string };
}) => {
  const { outputs, referenceOutputs } = params;
  const instructions = [
    "如果 ACTUAL 与 EXPECTED 在语义上等价，返回 1；否则返回 0。",
    "只返回 0 或 1，不要输出其他内容。",
  ].join("\n");
  const grade = await client.chat.completions.create({
    model: "gpt-4o-mini-2024-07-18",
    temperature: 0,
    messages: [
      { role: "system", content: instructions },
      { role: "user", content: `ACTUAL: ${outputs.sql}\nEXPECTED: ${referenceOutputs?.sql}` },
    ],
  });
  return { key: "correctness", score: parseInt(grade.choices[0].message.content ?? "", 10) || 0 };
};

// swap-pairwise.ts —— 交换协议 pairwise 判官
// 运行：OPENAI_API_KEY=sk-占位 npx tsx swap-pairwise.ts （需联网，付费）
type Verdict = "A" | "B" | "tie";
type Outcome = "A_win" | "B_win" | "position_split" | "double_tie";

async function pairwiseOnce(q: string, first: string, second: string): Promise<Verdict> {
  const r = await client.chat.completions.create({
    model: "gpt-4o-2024-08-06",          // pairwise 判别用强模型（锚点见 18.4.3）
    temperature: 0,
    response_format: { type: "json_object" },
    messages: [{
      role: "user",
      content:
        `问题：${q}\n\n[回答一]\n${first}\n\n[回答二]\n${second}\n\n` +
        `哪个更好？只输出 JSON：{"winner":"A"|"B"|"tie"}，A 指回答一，B 指回答二。`,
    }],
  });
  const w = JSON.parse(r.choices[0].message.content ?? "{}").winner;
  return w === "A" || w === "B" ? w : "tie";
}

export async function pairwise(q: string, answerA: string, answerB: string) {
  const [r1, r2] = await Promise.all([
    pairwiseOnce(q, answerA, answerB),   // 顺序一：A 在前
    pairwiseOnce(q, answerB, answerA),   // 顺序二：B 在前
  ]);
  // 一致性判定：四种结局的语义
  let outcome: Outcome;
  if (r1 === "A" && r2 === "B") outcome = "A_win";          // 两种顺序都指向 A → 采纳
  else if (r1 === "B" && r2 === "A") outcome = "B_win";     // 两种顺序都指向 B → 采纳
  else if (r1 === "tie" && r2 === "tie") outcome = "double_tie"; // 两次平局 → 平
  else outcome = "position_split";                          // 判决跟着位置走 → 丢弃
  return { winner: outcome === "A_win" ? "A" : outcome === "B_win" ? "B" : "tie", outcome };
}

// ensemble-judge.ts —— 三判官多数投票（成本 ×3，建议只用于高争议样本）
export async function ensembleJudge(
  q: string, a: string, b: string,
  judges: string[] = ["gpt-4o-2024-08-06", "claude-3-5-sonnet-20241022", "gemini-1.5-pro-002"],
): Promise<"A" | "B" | "tie"> {
  // pairwiseOnceWithModel = 18.4.2 的 pairwiseOnce 加一个 model 参数，逻辑完全一致
  const votes = await Promise.all(judges.map(m => pairwiseOnceWithModel(m, q, a, b)));
  const aWins = votes.filter(v => v === "A").length;
  const bWins = votes.filter(v => v === "B").length;
  return aWins > bWins ? "A" : bWins > aWins ? "B" : "tie"; // 平票 → 交人工
}

// cohens-kappa.ts —— 判官与人工标注的一致性验收
// 运行：npx tsx cohens-kappa.ts （无需联网）
export function cohensKappa(human: number[], judge: number[]) {
  if (human.length !== judge.length || human.length === 0) throw new Error("length mismatch");
  const n = human.length;
  const cats = [...new Set([...human, ...judge])];
  const po = human.filter((h, i) => h === judge[i]).length / n;   // 观察一致率
  const pe = cats.reduce((s, c) =>                                 // 机遇一致率
    s + (human.filter(x => x === c).length / n) * (judge.filter(x => x === c).length / n), 0);
  return { po, pe, kappa: pe === 1 ? 0 : (po - pe) / (1 - pe) };
}

// 例：10 条样本，人工与判官只在第 3、8 条上分歧
const human = [1, 1, 0, 0, 1, 1, 0, 1, 0, 0];
const judge = [1, 1, 0, 1, 1, 1, 0, 1, 1, 0];
console.log(cohensKappa(human, judge));
// 期望输出：{ po: 0.8, pe: 0.5, kappa: 0.6 }
// 读法：观察一致率 0.8 看着很高，扣掉机遇后真实超出一致的部分只剩 0.6

// length-drift.ts —— 冗长偏差探针：分数与长度的 Spearman 相关
// 运行：npx tsx length-drift.ts （无需联网）
function rank(a: number[]): number[] {
  return a.map((v, i) => [v, i] as const)
    .sort((p, q) => p[0] - q[0])
    .reduce<number[]>((r, [, i], k) => (r[i] = k + 1, r), []);
}
export function spearman(x: number[], y: number[]): number {
  const rx = rank(x), ry = rank(y), n = x.length;
  const mx = rx.reduce((s, v) => s + v, 0) / n, my = ry.reduce((s, v) => s + v, 0) / n;
  const cov = rx.reduce((s, v, i) => s + (v - mx) * (ry[i] - my), 0);
  const sx = Math.sqrt(rx.reduce((s, v) => s + (v - mx) ** 2, 0));
  const sy = Math.sqrt(ry.reduce((s, v) => s + (v - my) ** 2, 0));
  return cov / (sx * sy);
}
const scores  = [3, 4, 2, 5, 3, 4];            // 判官输出
const lengths = [180, 900, 150, 1200, 260, 860]; // 对应回答长度（字符数）
console.log(spearman(scores, lengths).toFixed(2));
// 期望输出：0.94 —— 强正相关：这个判官在给"长"加分，rubric 中立声明失效

// routing.ts —— 评分器路由：能用规则评的不进判官
export type Route = "rule" | "judge";
export function routeGrader(item: { kind: "math" | "code" | "fact" | "open" }): Route {
  if (item.kind === "math")  return "rule";  // 精确匹配或符号等价（SymPy 类）
  if (item.kind === "code")  return "rule";  // 单元测试
  if (item.kind === "fact")  return "rule";  // 参考答案 + 0/1 语义等价（18.3.3）
  return "judge";                            // 只有开放式产出进判官
}

// normalize.ts —— 判官输入的格式归一化：剥离 Markdown 修饰，只留内容
// 运行：npx tsx normalize.ts （无需联网）
export function stripMarkdown(s: string): string {
  return s
    .replace(/```[\s\S]*?```/g, "（代码块）")  // 代码块用占位符，防长度剧变
    .replace(/!\[[^\]]*\]\([^)]*\)/g, "")       // 图片
    .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")    // 链接只留文字
    .replace(/[*_#>]+/g, "")                     // 强调与标题符号
    .replace(/\n{3,}/g, "\n\n");
}

// two-stage-judge.ts —— 两段式判官：分流条件决定谁复核
// 依赖 18.3.2 的 judgeOnce 与 RUBRIC
const CHEAP = "gpt-4o-mini-2024-07-18";
const STRONG = "gpt-4o-2024-08-06";
const NEAR = 0.5; // 距阈值多近算"贴近"（本书经验值，按业务调）

export async function twoStageJudge(q: string, answer: string, ctx: string) {
  const cheap = await judgeOnce(CHEAP, RUBRIC_V1, q, answer, ctx);   // 第一段：全量初筛
  if (cheap.status !== "ok")                                          // 解析失败 → 复核
    return { escalated: true, final: await judgeOnce(STRONG, RUBRIC_V1, q, answer, ctx) };
  const near = Math.abs(cheap.total - GATE_V1.totalMin) <= NEAR;      // 贴近阈值 → 复核
  const risky = GATE_V1.blocking.some(d => (cheap.perDim[d] ?? 5) <= 3); // 阻断维度低分 → 复核
  const conflicted = !near && !risky && !(cheap.total >= GATE_V1.totalMin + 1); // 分数漂在中间带
  return near || risky || conflicted
    ? { escalated: true, final: await judgeOnce(STRONG, RUBRIC_V1, q, answer, ctx) }
    : { escalated: false, final: cheap };
}