Set up an LLM eval suite

[
  {
    "id": "1",
    "input": "Summarise this support ticket: ...",
    "expected": {
      "must_contain_any": ["refund", "return", "exchange"],
      "max_words": 80,
      "must_be_in_language": "en"
    }
  },
  {
    "id": "2",
    "input": "Translate this Arabic invoice description: ...",
    "expected": {
      "must_be_in_language": "ar",
      "must_not_contain": ["TODO", "[INSERT", "<", ">"]
    }
  }
]

import { OpenAI } from 'openai';

interface DatasetEntry {
  id: string;
  input: string;
  expected: Record<string, unknown>;
}

interface Result {
  id: string;
  output: string;
  expected: DatasetEntry['expected'];
}

const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

async function runOne(entry: DatasetEntry, prompt: string): Promise<Result> {
  const resp = await client.chat.completions.create({
    model: 'gpt-4o-mini',
    messages: [
      { role: 'system', content: prompt },
      { role: 'user', content: entry.input },
    ],
  });
  return {
    id: entry.id,
    output: resp.choices[0].message.content ?? '',
    expected: entry.expected,
  };
}

export async function runAll(
  dataset: DatasetEntry[],
  prompt: string,
  concurrency = 4,
): Promise<Result[]> {
  const results: Result[] = [];
  let cursor = 0;
  await Promise.all(
    Array(concurrency).fill(0).map(async () => {
      while (cursor < dataset.length) {
        const idx = cursor++;
        results[idx] = await runOne(dataset[idx], prompt);
      }
    }),
  );
  return results;
}

type Score = { ok: boolean; reason?: string };

const scorers = {
  must_contain_any: (out: string, expected: string[]): Score =>
    expected.some((s) => out.toLowerCase().includes(s.toLowerCase()))
      ? { ok: true }
      : { ok: false, reason: `none of: ${expected.join(', ')}` },

  must_not_contain: (out: string, expected: string[]): Score => {
    const hit = expected.find((s) => out.includes(s));
    return hit ? { ok: false, reason: `contains: ${hit}` } : { ok: true };
  },

  max_words: (out: string, expected: number): Score => {
    const n = out.trim().split(/\s+/).length;
    return n <= expected
      ? { ok: true }
      : { ok: false, reason: `${n} > ${expected} words` };
  },

  min_chars: (out: string, expected: number): Score =>
    out.length >= expected
      ? { ok: true }
      : { ok: false, reason: `${out.length} < ${expected} chars` },

  must_be_in_language: (out: string, expected: 'ar' | 'en'): Score => {
    const arabic = /[؀-ۿ]/.test(out);
    const isAr = arabic;
    return (expected === 'ar') === isAr
      ? { ok: true }
      : { ok: false, reason: `expected ${expected}, looks ${isAr ? 'ar' : 'en'}` };
  },
};

function scoreOne(result: Result): { id: string; pass: boolean; failures: string[] } {
  const failures: string[] = [];
  for (const [key, expected] of Object.entries(result.expected)) {
    const fn = scorers[key as keyof typeof scorers];
    if (!fn) continue;
    const s = fn(result.output as never, expected as never);
    if (!s.ok) failures.push(`${key}: ${s.reason}`);
  }
  return { id: result.id, pass: failures.length === 0, failures };
}

export function summarise(results: Result[]) {
  const scored = results.map(scoreOne);
  const passed = scored.filter((r) => r.pass).length;
  const total = scored.length;
  return {
    passRate: passed / total,
    failed: scored.filter((r) => !r.pass),
  };
}

- run: npm run eval -- --prompt prompts/v3.txt --threshold 0.85
- run: npm run eval -- --prompt prompts/v3.txt --diff prompts/v2.txt