How to Evaluate LLM Outputs: A Practical Guide to Building Evals

def exact_match(output: str, expected: str) -> float:
    return 1.0 if output.strip() == expected.strip() else 0.0

import numpy as np
from openai import OpenAI
 
client = OpenAI(base_url="https://api.aicredits.in/v1", api_key="sk-your-key")
 
def embed(text: str) -> list[float]:
    resp = client.embeddings.create(
        model="text-embedding-3-small",
        input=text,
    )
    return resp.data[0].embedding
 
def cosine_similarity(a: list[float], b: list[float]) -> float:
    a, b = np.array(a), np.array(b)
    return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
 
def semantic_score(output: str, expected: str) -> float:
    return cosine_similarity(embed(output), embed(expected))

import json
import time
from dataclasses import dataclass, field
from openai import OpenAI
 
client = OpenAI(
    base_url="https://api.aicredits.in/v1",
    api_key="sk-your-aicredits-key",
)
 
# --- Data types ---
 
@dataclass
class EvalExample:
    id: str
    input: str
    expected: str
    metadata: dict = field(default_factory=dict)
 
@dataclass
class EvalResult:
    id: str
    input: str
    expected: str
    output: str
    score: float
    latency_ms: float
    cost_tokens: int
    passed: bool
 
# --- Model under test ---
 
SYSTEM_PROMPT = """You are a customer support classifier.
Given a user message, respond with exactly one of these categories:
billing, technical, account, general
 
Respond with only the category name, nothing else."""
 
def run_model(user_input: str, model: str = "gpt-4o-mini") -> tuple[str, float, int]:
    """Returns (output, latency_ms, total_tokens)."""
    start = time.perf_counter()
    resp = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": user_input},
        ],
        temperature=0,
        max_tokens=16,
    )
    latency_ms = (time.perf_counter() - start) * 1000
    output = resp.choices[0].message.content.strip().lower()
    tokens = resp.usage.total_tokens
    return output, latency_ms, tokens
 
# --- Grader ---
 
def grade(output: str, expected: str) -> float:
    return exact_match(output, expected)
 
def exact_match(output: str, expected: str) -> float:
    return 1.0 if output == expected.lower().strip() else 0.0
 
# --- Harness ---
 
def run_eval(dataset: list[EvalExample], model: str = "gpt-4o-mini") -> list[EvalResult]:
    results = []
    for ex in dataset:
        output, latency_ms, tokens = run_model(ex.input, model=model)
        score = grade(output, ex.expected)
        results.append(EvalResult(
            id=ex.id,
            input=ex.input,
            expected=ex.expected,
            output=output,
            score=score,
            latency_ms=latency_ms,
            cost_tokens=tokens,
            passed=score >= 1.0,
        ))
    return results
 
def print_report(results: list[EvalResult]):
    total = len(results)
    passed = sum(r.passed for r in results)
    avg_latency = sum(r.latency_ms for r in results) / total
    total_tokens = sum(r.cost_tokens for r in results)
    accuracy = passed / total
 
    print(f"\n{'='*50}")
    print(f"Eval Results — {total} examples")
    print(f"{'='*50}")
    print(f"Accuracy:      {accuracy:.1%}  ({passed}/{total})")
    print(f"Avg latency:   {avg_latency:.0f} ms")
    print(f"Total tokens:  {total_tokens:,}")
    print(f"\nFailed cases:")
    for r in results:
        if not r.passed:
            print(f"  [{r.id}] input={r.input!r}")
            print(f"         expected={r.expected!r}  got={r.output!r}")
 
# --- Example dataset ---
 
dataset = [
    EvalExample("001", "My invoice is wrong", "billing"),
    EvalExample("002", "The API keeps timing out", "technical"),
    EvalExample("003", "I need to reset my password", "account"),
    EvalExample("004", "What are your business hours?", "general"),
    EvalExample("005", "I was charged twice this month", "billing"),
    EvalExample("006", "How do I integrate with Python?", "technical"),
]
 
if __name__ == "__main__":
    results = run_eval(dataset)
    print_report(results)

JUDGE_PROMPT = """You are an expert evaluator for a customer support AI.
 
Your job is to assess whether the AI's response correctly addresses the user's question.
 
Rubric:
5 — Fully correct: accurate, complete, and clearly addresses the question
4 — Mostly correct: minor omissions or slight inaccuracies, but helpful overall
3 — Partially correct: addresses part of the question but misses key points
2 — Mostly incorrect: the response is largely unhelpful or misleading
1 — Completely wrong: factually incorrect or entirely off-topic
 
Respond in this exact format:
<reasoning>One to three sentences explaining your score.</reasoning>
<score>N</score>
 
User question: {question}
AI response: {response}
Reference answer: {reference}"""
 
import re
 
def llm_judge(question: str, response: str, reference: str,
              judge_model: str = "gpt-4o-mini") -> tuple[float, str]:
    prompt = JUDGE_PROMPT.format(
        question=question,
        response=response,
        reference=reference,
    )
    result = client.chat.completions.create(
        model=judge_model,
        messages=[{"role": "user", "content": prompt}],
        temperature=0,
        max_tokens=256,
    )
    text = result.choices[0].message.content
 
    score_match = re.search(r"<score>(\d)</score>", text)
    reasoning_match = re.search(r"<reasoning>(.*?)</reasoning>", text, re.DOTALL)
 
    score = int(score_match.group(1)) / 5.0 if score_match else 0.0
    reasoning = reasoning_match.group(1).strip() if reasoning_match else ""
    return score, reasoning

# Save current score
python eval.py --prompt prompts/v1.txt --output results/v1.json
 
# Edit your prompt...
 
# Score the new version
python eval.py --prompt prompts/v2.txt --output results/v2.json
 
# Compare
python compare.py results/v1.json results/v2.json

import json, sys
 
def load(path):
    with open(path) as f:
        return json.load(f)
 
def compare(path_a, path_b):
    a = load(path_a)
    b = load(path_b)
    acc_a = sum(r["passed"] for r in a) / len(a)
    acc_b = sum(r["passed"] for r in b) / len(b)
    delta = acc_b - acc_a
    symbol = "+" if delta >= 0 else ""
    print(f"Accuracy: {acc_a:.1%} -> {acc_b:.1%}  ({symbol}{delta:.1%})")
 
    # Show examples that changed
    by_id_a = {r["id"]: r for r in a}
    regressions = [r for r in b if not r["passed"] and by_id_a.get(r["id"], {}).get("passed")]
    improvements = [r for r in b if r["passed"] and not by_id_a.get(r["id"], {}).get("passed")]
 
    if regressions:
        print(f"\nRegressions ({len(regressions)}):")
        for r in regressions:
            print(f"  [{r['id']}] {r['input']!r}")
    if improvements:
        print(f"\nImprovements ({len(improvements)}):")
        for r in improvements:
            print(f"  [{r['id']}] {r['input']!r}")
 
compare(sys.argv[1], sys.argv[2])

# .github/workflows/evals.yml
name: LLM Evals
 
on:
  pull_request:
    paths:
      - "prompts/**"
      - "src/llm/**"
 
jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install openai numpy
      - run: python eval.py --threshold 0.85
        env:
          OPENAI_API_KEY: ${{ secrets.AICREDITS_API_KEY }}
          OPENAI_BASE_URL: "https://api.aicredits.in/v1"

# gpt-4o-mini pricing (approximate)
INPUT_COST_PER_1M = 0.15   # USD
OUTPUT_COST_PER_1M = 0.60  # USD
 
examples = 100
avg_input_tokens = 150   # prompt + user message
avg_output_tokens = 10   # short structured output
 
input_cost = (examples * avg_input_tokens / 1_000_000) * INPUT_COST_PER_1M
output_cost = (examples * avg_output_tokens / 1_000_000) * OUTPUT_COST_PER_1M
total_usd = input_cost + output_cost
 
print(f"Cost per eval run: ${total_usd:.4f}")
# -> Cost per eval run: $0.0023

client = OpenAI(
    base_url="https://api.aicredits.in/v1",
    api_key="sk-your-aicredits-key",  # prepaid INR wallet, no USD card needed
)

import json
import time
import re
from dataclasses import dataclass, field
from openai import OpenAI
 
client = OpenAI(
    base_url="https://api.aicredits.in/v1",
    api_key="sk-your-aicredits-key",
)
 
# Prompt under test
SYSTEM_PROMPT = """Classify the customer message into exactly one category:
billing, technical, account, general
 
Rules:
- billing: payment issues, invoices, refunds, subscription charges
- technical: API errors, bugs, integration problems, performance issues
- account: login, password, profile settings, access permissions
- general: pricing questions, product info, business hours, other
 
Respond with only the category name in lowercase."""
 
DATASET = [
    {"id": "001", "input": "I was charged twice for my subscription", "expected": "billing"},
    {"id": "002", "input": "The API returns 429 errors under load", "expected": "technical"},
    {"id": "003", "input": "How do I change my email address?", "expected": "account"},
    {"id": "004", "input": "Do you offer a free trial?", "expected": "general"},
    {"id": "005", "input": "My credit card was declined", "expected": "billing"},
    {"id": "006", "input": "Embeddings endpoint is timing out", "expected": "technical"},
    {"id": "007", "input": "I forgot my password", "expected": "account"},
    {"id": "008", "input": "What models do you support?", "expected": "general"},
    {"id": "009", "input": "I need a refund for last month", "expected": "billing"},
    {"id": "010", "input": "Getting a 401 on every request", "expected": "technical"},
    {"id": "011", "input": "Can I add a team member to my account?", "expected": "account"},
    {"id": "012", "input": "Is there a Python SDK?", "expected": "technical"},
    {"id": "013", "input": "My invoice shows the wrong amount", "expected": "billing"},
    {"id": "014", "input": "How do rate limits work?", "expected": "general"},
    {"id": "015", "input": "I cannot log in after changing my password", "expected": "account"},
]
 
VALID_CATEGORIES = {"billing", "technical", "account", "general"}
 
def run_example(ex: dict, model: str = "gpt-4o-mini") -> dict:
    start = time.perf_counter()
    resp = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": ex["input"]},
        ],
        temperature=0,
        max_tokens=16,
    )
    latency_ms = (time.perf_counter() - start) * 1000
    output = resp.choices[0].message.content.strip().lower()
    tokens = resp.usage.total_tokens
 
    exact = output == ex["expected"]
    valid_format = output in VALID_CATEGORIES
 
    return {
        "id": ex["id"],
        "input": ex["input"],
        "expected": ex["expected"],
        "output": output,
        "passed": exact,
        "format_ok": valid_format,
        "latency_ms": round(latency_ms, 1),
        "tokens": tokens,
    }
 
def run_eval(dataset: list[dict], model: str = "gpt-4o-mini") -> list[dict]:
    print(f"Running eval on {len(dataset)} examples with model={model}")
    return [run_example(ex, model=model) for ex in dataset]
 
def report(results: list[dict]):
    n = len(results)
    accuracy = sum(r["passed"] for r in results) / n
    format_rate = sum(r["format_ok"] for r in results) / n
    avg_latency = sum(r["latency_ms"] for r in results) / n
    total_tokens = sum(r["tokens"] for r in results)
 
    print(f"\n{'='*55}")
    print(f"{'Metric':<25} {'Value':>10}")
    print(f"{'='*55}")
    print(f"{'Accuracy':<25} {accuracy:>9.1%}")
    print(f"{'Format compliance':<25} {format_rate:>9.1%}")
    print(f"{'Avg latency':<25} {avg_latency:>8.0f}ms")
    print(f"{'Total tokens':<25} {total_tokens:>10,}")
    print(f"{'='*55}")
 
    failures = [r for r in results if not r["passed"]]
    if failures:
        print(f"\nFailed ({len(failures)}/{n}):")
        for r in failures:
            mark = "BAD_FORMAT" if not r["format_ok"] else "WRONG"
            print(f"  [{r['id']}] {mark}: {r['input']!r}")
            print(f"         expected={r['expected']!r}  got={r['output']!r}")
    else:
        print("\nAll examples passed.")
 
if __name__ == "__main__":
    results = run_eval(DATASET)
    report(results)
 
    with open("eval_results.json", "w") as f:
        json.dump(results, f, indent=2)
    print("\nResults saved to eval_results.json")