29. 案例研究（三）：拍照解题 App 的多模态评估

// photo-solve/types.ts —— 流水线各层的类型契约与预算（无需联网）
// 每层的输出都是显式结构，评估瞄准观测点而不是"最终答案对不对"

export interface OcrResult {       // 第 1 层：文字识别
  text: string;
  language: "zh" | "en";
  confidence: number;
}

export interface FormulaResult {   // 第 2 层：公式结构化
  latex: string;                   // 规范化后的 LaTeX
  isFormula: boolean;              // 该片段是否是公式
}

export interface SolveResult {     // 第 3+4 层：推理与讲解
  answer: string;
  steps: string[];
}

export interface PipelineTrace extends Record<string, unknown> {
  imageHash: string;               // 图片哈希：评估缓存与坏例定位的钥匙
  ocr: OcrResult;
  formulas: FormulaResult[];
  solve: SolveResult;
  latencyMs: { ocr: number; solve: number; total: number };
  costUsd: number;
}

// 预算断言：管道层有延迟与成本预算，评估 harness 里强制执行（第 20 章 17.6.4 的思想）
export const BUDGET = {
  ocrP95Ms: 800,
  solveP95Ms: 2200,
  totalP95Ms: 3000,
  costPerQueryUsd: 0.01,
} as const;

// photo-solve/gen-geometry.ts —— 几何合成数据：参数 → SVG + 真值（本地运行，无需联网）
// 运行：npx tsx photo-solve/gen-geometry.ts 200 data/geometry-syn.jsonl
import { writeFileSync } from "node:fs";

interface GeoSample {
  id: string;
  svg: string;                 // 渲染出的题目图（可转 PNG 供模型输入）
  kind: "triangle-angle" | "rectangle-area";
  expected: string;            // 真值答案，来自参数本身
}

function triangleAngle(seed: number): GeoSample {
  // 固定种子保证可复现：同一 seed 永远生成同一道题（第 4 章 3.7 可复现性）
  let s = seed;
  const rand = (lo: number, hi: number) => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648 * (hi - lo) + lo;
  const a = Math.round(rand(30, 80)), b = Math.round(rand(30, 80));
  const c = 180 - a - b;                       // 三角形内角和：真值由构造保证
  const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="300" height="240">
  <polygon points="40,200 260,200 ${40 + 200 * Math.cos((a * Math.PI) / 180)},${200 - 200 * Math.sin((a * Math.PI) / 180)}"
    fill="none" stroke="black" stroke-width="2"/>
  <text x="120" y="225" font-size="16">已知角 A = ${a}°，角 B = ${b}°，求角 C</text></svg>`;
  return { id: `geo-syn-${seed}`, svg, kind: "triangle-angle", expected: `${c}` };
}

const n = Number(process.argv[2] ?? 200);
const outPath = process.argv[3] ?? "data/geometry-syn.jsonl";
const samples = Array.from({ length: n }, (_, i) => triangleAngle(1000 + i));
writeFileSync(outPath, samples.map(s => JSON.stringify(s)).join("\n"));
console.log(`generated ${samples.length} samples → ${outPath}`);

// photo-solve/ocr-f1.ts —— 字符级 F1（含 CJK 处理；本地运行，无需联网）
// 运行：npx tsx photo-solve/ocr-f1.ts data/ocr-cases.jsonl
import { readFileSync } from "node:fs";

export function normalizeOcr(s: string): string {
  return s
    .replace(/\s+/g, "")                       // 空白不影响内容
    .replace(/[０-９]/g, (c) => String.fromCharCode(c.charCodeAt(0) - 0xfee0)) // 全角数字→半角
    .replace(/[？?！!。，,]/g, "")              // 标点归一
    .toLowerCase();
}

export function charF1(predicted: string, expected: string): number {
  const p = [...normalizeOcr(predicted)];      // 展开 Unicode 码点：CJK 必须按字符而不是字节
  const e = [...normalizeOcr(expected)];
  const bag = new Map<string, number>();
  for (const ch of e) bag.set(ch, (bag.get(ch) ?? 0) + 1);
  let tp = 0;
  for (const ch of p) {
    const left = bag.get(ch) ?? 0;
    if (left > 0) { tp++; bag.set(ch, left - 1); }
  }
  // 多重集比对：出现 2 次的字符只匹配 2 次（朴素集合会把重复字符判对）
  const precision = tp / Math.max(p.length, 1);
  const recall = tp / Math.max(e.length, 1);
  return precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
}

const rows = readFileSync(process.argv[2] ?? "data/ocr-cases.jsonl", "utf-8")
  .trim().split("\n").map(JSON.parse as (l: string) => { id: string; predicted: string; expected: string; language: string });

for (const lang of ["zh", "en"]) {
  const sub = rows.filter(r => r.language === lang);
  if (sub.length === 0) continue;
  const avg = sub.reduce((s, r) => s + charF1(r.predicted, r.expected), 0) / sub.length;
  console.log(`lang=${lang} charF1=${avg.toFixed(4)} n=${sub.length}`);
}
// 期望输出示例：lang=zh charF1=0.9387 n=350   lang=en charF1=0.9712 n=150

// photo-solve/latex-equiv.ts —— 公式等价两级判定（本地运行；npm i mathjs）
// 运行：npx tsx photo-solve/latex-equiv.ts
import { parse, simplify, evaluate } from "mathjs";

export function normalizeLatex(latex: string): string {
  return latex
    .replace(/\s+/g, "")
    .replace(/\\left|\\right/g, "")
    .replace(/\\cdot|\\times/g, "*")
    .replace(/\\div/g, "/")
    .replace(/[{}]/g, "");
}

// LaTeX 朴素子集 → mathjs 表达式（\frac{a}{b} → (a)/(b)，^ 保留，\sqrt{a} → sqrt(a)）
function latexToExpr(latex: string): string {
  return normalizeLatex(latex)
    .replace(/\\frac\{([^{}]+)\}\{([^{}]+)\}/g, "($1)/($2)")
    .replace(/\\sqrt\{([^{}]+)\}/g, "sqrt($1)")
    .replace(/\\pi/g, "pi");
}

export function latexEquivalent(a: string, b: string): { equivalent: boolean; method: string } {
  const na = normalizeLatex(a), nb = normalizeLatex(b);
  if (na === nb) return { equivalent: true, method: "normalize" };   // 第一级：规范化相等

  try {                                                              // 第二级：符号化简
    const ea = simplify(parse(latexToExpr(a)));
    const eb = simplify(parse(latexToExpr(b)));
    const diff = simplify(parse(`(${latexToExpr(a)})-(${latexToExpr(b)})`));
    if (diff.toString() === "0" || ea.equals(eb)) return { equivalent: true, method: "symbolic" };
  } catch { /* 符号化简不支持的表达式 → 走数值探测 */ }

  try {                                                              // 第三级：数值探测
    for (const x of [0.37, 1.21, 2.53, 4.71]) {                      // 固定采样点保证可复现
      const va = evaluate(latexToExpr(a).replace(/(^|[^a-z])x/g, `$1(${x})`));
      const vb = evaluate(latexToExpr(b).replace(/(^|[^a-z])x/g, `$1(${x})`));
      if (Math.abs(va - vb) > 1e-9 * Math.max(1, Math.abs(va))) return { equivalent: false, method: "numeric" };
    }
    return { equivalent: true, method: "numeric" };
  } catch {
    return { equivalent: false, method: "unparseable" };             // 解析不了按不等价，并记入错误率
  }
}

console.log(latexEquivalent("\\frac{1}{2}", "0.5"));                  // true（符号级）
console.log(latexEquivalent("x^2+2x+1", "(x+1)^2"));                  // true（数值探测）
console.log(latexEquivalent("\\frac{a}{b}", "\\frac{b}{a}"));         // false——字符串最像的一对

// photo-solve/pope.ts —— POPE 式幻觉检测（需联网 + API 费用）
// 运行：export OPENAI_API_KEY=sk-你的密钥 && npx tsx photo-solve/pope.ts data/pope.jsonl
import OpenAI from "openai";
import { readFileSync } from "node:fs";

if (!process.env.OPENAI_API_KEY) throw new Error("请先设置环境变量 OPENAI_API_KEY");
const openai = new OpenAI();

interface PopeItem { id: string; imageB64: string; question: string; objectPresent: boolean; }

export async function runPope(items: PopeItem[]) {
  let tp = 0, fp = 0, tn = 0, fn = 0, yes = 0;
  for (const it of items) {
    const r = await openai.chat.completions.create({
      model: "gpt-4o-mini",
      temperature: 0,
      messages: [{
        role: "user",
        content: [
          { type: "text", text: `${it.question} 只回答 yes 或 no。` },
          { type: "image_url", image_url: { url: `data:image/png;base64,${it.imageB64}` } },
        ],
      }],
    });
    const said = /yes/i.test(r.choices[0].message.content ?? "") ;
    if (said) yes++;
    if (said && it.objectPresent) tp++;
    else if (said && !it.objectPresent) fp++;   // 幻觉：图中无物说有
    else if (!said && !it.objectPresent) tn++;
    else fn++;
  }
  const precision = tp / Math.max(tp + fp, 1);
  const recall = tp / Math.max(tp + fn, 1);
  return {
    f1: (2 * precision * recall) / Math.max(precision + recall, 1e-9),
    hallucinationRate: fp / Math.max(fp + tn, 1), // "无中生有"率：产品最该盯的数字
    yesRate: yes / items.length,                  // 防模型"逢问必 yes"的均衡哨兵
  };
}

const items: PopeItem[] = readFileSync(process.argv[2] ?? "data/pope.jsonl", "utf-8")
  .trim().split("\n").map(JSON.parse);
console.log(await runPope(items));
// 期望输出示例：{ f1: 0.912, hallucinationRate: 0.043, yesRate: 0.51 }

// photo-solve/budget.ts —— 预算断言：超预算的 run 直接失败（本地运行，无需联网）
import { BUDGET, type PipelineTrace } from "./types.ts";

export function assertBudget(traces: PipelineTrace[]): void {
  const p95 = (xs: number[]) => xs.sort((a, b) => a - b)[Math.floor(xs.length * 0.95)] ?? Infinity;
  const violations: string[] = [];
  if (p95(traces.map(t => t.latencyMs.ocr)) > BUDGET.ocrP95Ms) violations.push("ocr P95 超预算");
  if (p95(traces.map(t => t.latencyMs.solve)) > BUDGET.solveP95Ms) violations.push("solve P95 超预算");
  if (p95(traces.map(t => t.latencyMs.total)) > BUDGET.totalP95Ms) violations.push("端到端 P95 超预算");
  const avgCost = traces.reduce((s, t) => s + t.costUsd, 0) / traces.length;
  if (avgCost > BUDGET.costPerQueryUsd) violations.push(`平均成本 $${avgCost.toFixed(4)} 超预算`);
  if (violations.length) { console.error("预算违例：" + violations.join("；")); process.exitCode = 1; }
  else console.log("预算内 ✓");
}