Prompt Engineering Testing: Building Regression Suites for Prompt Changes

# evals/golden_dataset.py
"""
Golden dataset for customer support chatbot.
Run: python -m pytest tests/evals/ -v
"""

GOLDEN_DATASET = [
    {
        "id": "pricing-001",
        "category": "pricing",
        "input": "How much does HelpMeTest cost?",
        "context": "HelpMeTest uses usage-based pricing: $0.003 per test run ($3 per 1,000 runs), no base fee.",
        "expected_behavior": "Mentions usage-based pricing and the per-run rate",
        "required_elements": ["$0.003", "run"],
        "forbidden_elements": ["free tier", "enterprise"],  # Should not conflate plans
    },
    {
        "id": "pricing-002",
        "category": "pricing",
        "input": "Do you have a free tier?",
        "context": "No free tier. A 14-day free trial is available (card required, not charged if cancelled before the trial ends).",
        "expected_behavior": "Mentions the 14-day trial and that a card is required",
        "required_elements": ["trial", "card"],
        "forbidden_elements": ["free tier", "no credit card"],
    },
    {
        "id": "scope-001",
        "category": "out-of-scope",
        "input": "Can you write my Python code for me?",
        "context": "",
        "expected_behavior": "Politely declines, focuses on testing help",
        "required_elements": [],
        "forbidden_elements": ["here is", "def ", "import"],  # Should not write code
    },
    {
        "id": "format-001",
        "category": "format",
        "input": "List all the features of HelpMeTest's pricing",
        "context": "Usage-based plan: unlimited tests, parallel execution, 6-month data retention.",
        "expected_behavior": "Uses a list format",
        "required_elements": [],
        "format_check": "contains_list",  # Checks for bullet points or numbered list
    },
]

# evals/evaluator.py
import json
import re
from dataclasses import dataclass
from typing import Callable

@dataclass
class EvalResult:
    test_id: str
    passed: bool
    score: float
    details: dict

class PromptEvaluator:
    def __init__(self, llm_fn: Callable[[str, str], str]):
        """
        llm_fn: function(prompt, context) -> str
        The function wrapping your LLM with the current prompt.
        """
        self.llm_fn = llm_fn
    
    def evaluate_case(self, case: dict) -> EvalResult:
        actual_output = self.llm_fn(case["input"], case.get("context", ""))
        
        checks = []
        
        # Check required elements
        for required in case.get("required_elements", []):
            present = required.lower() in actual_output.lower()
            checks.append(("required_element", required, present))
        
        # Check forbidden elements
        for forbidden in case.get("forbidden_elements", []):
            absent = forbidden.lower() not in actual_output.lower()
            checks.append(("forbidden_element", forbidden, absent))
        
        # Format checks
        if case.get("format_check") == "contains_list":
            has_list = bool(re.search(r'(\n[-•*]\s|\n\d+\.\s)', actual_output))
            checks.append(("format_check", "contains_list", has_list))
        
        passed_checks = sum(1 for _, _, result in checks if result)
        total_checks = len(checks) if checks else 1
        score = passed_checks / total_checks
        passed = score >= 1.0  # All checks must pass
        
        return EvalResult(
            test_id=case["id"],
            passed=passed,
            score=score,
            details={
                "actual_output": actual_output,
                "checks": [{"type": t, "target": target, "passed": r} for t, target, r in checks],
            }
        )
    
    def evaluate_dataset(self, dataset: list[dict]) -> dict:
        results = [self.evaluate_case(case) for case in dataset]
        
        by_category = {}
        for result, case in zip(results, dataset):
            cat = case.get("category", "uncategorized")
            if cat not in by_category:
                by_category[cat] = []
            by_category[cat].append(result)
        
        return {
            "total": len(results),
            "passed": sum(1 for r in results if r.passed),
            "overall_score": sum(r.score for r in results) / len(results),
            "by_category": {
                cat: {
                    "passed": sum(1 for r in cat_results if r.passed),
                    "total": len(cat_results),
                    "score": sum(r.score for r in cat_results) / len(cat_results),
                }
                for cat, cat_results in by_category.items()
            },
            "failures": [
                {"id": r.test_id, "score": r.score, "output": r.details["actual_output"][:200]}
                for r in results if not r.passed
            ],
        }

# tests/evals/test_prompt_regression.py
import pytest
import json
import os
from pathlib import Path

from evals.golden_dataset import GOLDEN_DATASET
from evals.evaluator import PromptEvaluator
from src.chatbot import ChatBot  # Your LLM wrapper

@pytest.fixture(scope="module")
def evaluator():
    bot = ChatBot(model="gpt-4o-mini")
    return PromptEvaluator(llm_fn=bot.respond)

@pytest.fixture(scope="module")
def eval_results(evaluator):
    return evaluator.evaluate_dataset(GOLDEN_DATASET)

def test_overall_score_above_baseline(eval_results):
    """Overall score must not drop more than 5% from baseline."""
    baseline_path = Path("evals/baseline_scores.json")
    
    if not baseline_path.exists():
        pytest.skip("No baseline recorded yet. Run: python evals/record_baseline.py")
    
    with open(baseline_path) as f:
        baseline = json.load(f)
    
    current_score = eval_results["overall_score"]
    baseline_score = baseline["overall_score"]
    
    assert current_score >= baseline_score - 0.05, (
        f"Overall score regressed: {current_score:.3f} vs baseline {baseline_score:.3f}\n"
        f"Failures: {eval_results['failures']}"
    )

def test_pricing_category_score(eval_results):
    """Pricing questions are business-critical — require 100% pass rate."""
    pricing = eval_results["by_category"].get("pricing", {})
    
    if not pricing:
        pytest.skip("No pricing test cases in dataset")
    
    assert pricing["passed"] == pricing["total"], (
        f"Pricing accuracy degraded: {pricing['passed']}/{pricing['total']} passed"
    )

def test_out_of_scope_handling(eval_results):
    """Out-of-scope requests must be rejected correctly."""
    oos = eval_results["by_category"].get("out-of-scope", {})
    
    if not oos:
        pytest.skip("No out-of-scope test cases")
    
    assert oos["score"] >= 0.9, (
        f"Out-of-scope handling degraded: score {oos['score']:.2f}"
    )

@pytest.mark.parametrize("test_case", GOLDEN_DATASET)
def test_individual_case(test_case, evaluator):
    """Parametrized: run each golden case as an individual test."""
    result = evaluator.evaluate_case(test_case)
    
    assert result.passed, (
        f"Test {test_case['id']} failed (score: {result.score:.2f})\n"
        f"Input: {test_case['input']}\n"
        f"Output: {result.details['actual_output'][:300]}\n"
        f"Failed checks: {[c for c in result.details['checks'] if not c['passed']]}"
    )

# src/prompts/customer_support/v1.py
SYSTEM_PROMPT_V1 = """You are a helpful customer support agent for HelpMeTest.
Answer questions about our product using only the provided context.
Be concise and friendly."""

# src/prompts/customer_support/v2.py  
SYSTEM_PROMPT_V2 = """You are a helpful customer support agent for HelpMeTest.
Answer questions using only the provided context.
Always mention relevant pricing tiers when discussing features.
Format lists as bullet points.
Be concise and professional."""

# tests/evals/test_prompt_v2_regression.py
"""
Regression test for v2 prompt changes.
V2 adds: pricing tier mentions, bullet point formatting.
Run before merging prompt v2 changes.
"""
import pytest
from evals.golden_dataset import GOLDEN_DATASET
from evals.evaluator import PromptEvaluator
from src.chatbot import ChatBot

def test_v2_does_not_regress_v1_scores():
    """V2 must score >= V1 on the existing golden dataset."""
    v1_bot = ChatBot(model="gpt-4o-mini", prompt_version="v1")
    v2_bot = ChatBot(model="gpt-4o-mini", prompt_version="v2")
    
    v1_evaluator = PromptEvaluator(llm_fn=v1_bot.respond)
    v2_evaluator = PromptEvaluator(llm_fn=v2_bot.respond)
    
    v1_results = v1_evaluator.evaluate_dataset(GOLDEN_DATASET)
    v2_results = v2_evaluator.evaluate_dataset(GOLDEN_DATASET)
    
    assert v2_results["overall_score"] >= v1_results["overall_score"] - 0.05, (
        f"V2 prompt regressed: "
        f"v1={v1_results['overall_score']:.3f}, v2={v2_results['overall_score']:.3f}"
    )

def test_v2_improves_format_compliance():
    """V2 specifically aims to improve list formatting — verify this."""
    v1_bot = ChatBot(model="gpt-4o-mini", prompt_version="v1")
    v2_bot = ChatBot(model="gpt-4o-mini", prompt_version="v2")
    
    format_cases = [c for c in GOLDEN_DATASET if c.get("format_check")]
    
    v1_eval = PromptEvaluator(llm_fn=v1_bot.respond)
    v2_eval = PromptEvaluator(llm_fn=v2_bot.respond)
    
    v1_format_score = v1_eval.evaluate_dataset(format_cases)["overall_score"]
    v2_format_score = v2_eval.evaluate_dataset(format_cases)["overall_score"]
    
    assert v2_format_score > v1_format_score, (
        f"V2 should improve format compliance but didn't: "
        f"v1={v1_format_score:.3f}, v2={v2_format_score:.3f}"
    )

# .github/workflows/prompt-regression.yml
name: Prompt Regression Tests

on:
  pull_request:
    paths:
      - 'src/prompts/**'
      - 'src/chatbot.py'

jobs:
  prompt-regression:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      
      - name: Install dependencies
        run: pip install -r requirements-test.txt
      
      - name: Run prompt regression tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          pytest tests/evals/ \
            -v \
            --tb=short \
            --junitxml=results/prompt-regression.xml
      
      - name: Check score regression vs main
        if: always()
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          # Score current branch
          python evals/score_current.py --output results/current_scores.json
          
          # Compare against baseline (from main branch artifact)
          python evals/check_regression.py \
            --baseline results/baseline_scores.json \
            --current results/current_scores.json \
            --max-regression 0.05 \
            --fail-on-regression
      
      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: prompt-regression-results
          path: results/

# After verifying a prompt improvement is real:
python evals/record_baseline.py --commit  # Records current scores as new baseline

# This command should:
# 1. Run full eval suite
# 2. Write scores to evals/baseline_scores.json
# 3. Commit the updated baseline (automated via CI)

# scripts/model_migration_check.py
"""
Run before migrating to a new model version.
Compares old model + current prompts vs new model + current prompts.
"""

import argparse
from evals.golden_dataset import GOLDEN_DATASET
from evals.evaluator import PromptEvaluator
from src.chatbot import ChatBot

def compare_models(old_model: str, new_model: str):
    old_eval = PromptEvaluator(ChatBot(model=old_model).respond)
    new_eval = PromptEvaluator(ChatBot(model=new_model).respond)
    
    old_results = old_eval.evaluate_dataset(GOLDEN_DATASET)
    new_results = new_eval.evaluate_dataset(GOLDEN_DATASET)
    
    print(f"\n{'='*60}")
    print(f"Model Migration: {old_model} → {new_model}")
    print(f"{'='*60}")
    print(f"Overall: {old_results['overall_score']:.3f} → {new_results['overall_score']:.3f}")
    
    for category, old_cat in old_results["by_category"].items():
        new_cat = new_results["by_category"].get(category, {})
        change = new_cat.get("score", 0) - old_cat["score"]
        print(f"  {category}: {old_cat['score']:.3f} → {new_cat.get('score', 0):.3f} ({change:+.3f})")
    
    if new_results["failures"]:
        print(f"\nNew failures ({len(new_results['failures'])}):")
        for failure in new_results["failures"]:
            print(f"  - {failure['id']}: {failure['output'][:100]}...")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--old", default="gpt-4-turbo")
    parser.add_argument("--new", default="gpt-4o")
    args = parser.parse_args()
    
    compare_models(args.old, args.new)