CodeRabbit Integration Testing: PR Review Automation and Custom Rules Validation

# Create a branch with known bugs for testing CodeRabbit detection
git checkout -b coderabbit-test/known-issues

# Introduce specific issues CodeRabbit should catch
cat >> src/auth.js << 'EOF'

// Test: Should CodeRabbit catch hardcoded credentials?
const API_KEY = "sk-prod-abc123def456";

// Test: Should CodeRabbit catch SQL injection?
function getUserById(id) {
  return db.query(`SELECT * FROM users WHERE id = ${id}`);
}

// Test: Should CodeRabbit catch unhandled promise?
async function fetchData() {
  const response = await fetch('/api/data');
  return response.json(); // Missing error handling
}
EOF

git add . && git commit -m "test: known issues for CodeRabbit validation"
git push origin coderabbit-test/known-issues

# .coderabbit.yaml
reviews:
  profile: chill  # or assertive
  request_changes_workflow: false

  path_filters:
    - "!**/*.test.js"    # Don't review test files
    - "!**/node_modules" # Ignore dependencies

  path_instructions:
    - path: "src/api/**"
      instructions: |
        Check for:
        - Missing rate limiting on public endpoints
        - Endpoints that return user data without authentication check
        - Missing input validation

    - path: "src/payments/**"
      instructions: |
        This code handles billing. Apply maximum scrutiny:
        - Verify all amounts are handled as integers (cents), never floats
        - Check that idempotency keys are used for all Stripe API calls
        - Flag any logging of payment data (PCI compliance)

chat:
  auto_reply: true

// Create this file in a test PR to verify the rule fires
// src/payments/charge.js

// Should trigger: using float for money
function calculateTotal(price, quantity) {
  return price * quantity; // Floating point: $10.99 * 3 = $32.97000000000001
}

// Should trigger: logging payment data
function processPayment(card, amount) {
  console.log('Processing payment', { card, amount }); // PCI violation
  return stripe.charges.create({ amount, source: card.token });
}

// Should trigger: no idempotency key
async function chargeCustomer(customerId, amount) {
  return await stripe.charges.create({
    amount,
    currency: 'usd',
    customer: customerId,
    // Missing: idempotency_key
  });
}

# CodeRabbit Review Quality Log

## PR #123 (2024-03-15)
**CodeRabbit comments:** 8
**True positives:** 5 (real issues: 2 bugs, 3 style)
**False positives:** 3 (flagged valid patterns as issues)
**Known issues missed:** 1 (missing input validation, not caught)

False positive rate: 3/8 = 37.5% ← Too high, need to tune config

# Test 1: Verify CodeRabbit reviews happen automatically
# Create a PR and check that CodeRabbit comments within 2-3 minutes

# Test 2: Verify @coderabbitai commands work
# In a PR comment, type:
@coderabbitai review
# Should trigger a new review pass

@coderabbitai help
# Should return command list

@coderabbitai ignore
# Should suppress further comments on this PR

# Test 3: Verify review summary appears
# CodeRabbit should post a walkthrough comment summarizing the PR

# Create a PR with issues CodeRabbit should block
# Verify the PR is blocked
# Fix the issues
# Verify CodeRabbit re-reviews and unblocks

# .github/workflows/coderabbit-config-test.yml
name: Validate CodeRabbit Config

on:
  pull_request:
    paths:
      - '.coderabbit.yaml'

jobs:
  validate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Validate YAML syntax
        run: |
          python3 -c "import yaml; yaml.safe_load(open('.coderabbit.yaml'))"
          echo "Config syntax valid"

      - name: Check required fields
        run: |
          python3 << 'EOF'
          import yaml

          with open('.coderabbit.yaml') as f:
            config = yaml.safe_load(f)

          assert 'reviews' in config, "Missing reviews section"
          assert 'profile' in config['reviews'], "Missing reviews.profile"

          # Verify sensitive paths have instructions
          instructions = config.get('reviews', {}).get('path_instructions', [])
          paths = [i['path'] for i in instructions]

          required_sensitive_paths = ['src/payments/**', 'src/auth/**']
          for path in required_sensitive_paths:
            assert path in paths, f"Missing instructions for {path}"

          print("Config validation passed")
          EOF

# review_quality_tracker.py
# Run weekly, pull from GitHub API

import requests

REPO = "your-org/your-repo"
TOKEN = "your-github-token"

headers = {"Authorization": f"token {TOKEN}"}

# Get PRs from last 30 days
prs = requests.get(
    f"https://api.github.com/repos/{REPO}/pulls",
    params={"state": "closed", "per_page": 100},
    headers=headers
).json()

for pr in prs:
    comments = requests.get(
        f"https://api.github.com/repos/{REPO}/pulls/{pr['number']}/comments",
        headers=headers
    ).json()

    coderabbit_comments = [
        c for c in comments
        if c['user']['login'] == 'coderabbitai[bot]'
    ]

    # Track: how many CodeRabbit comments led to code changes?
    # (Heuristic: if the PR was updated after the CodeRabbit comment, it was likely addressed)
    print(f"PR #{pr['number']}: {len(coderabbit_comments)} CodeRabbit comments")

# Fix: exclude generated and vendored files
reviews:
  path_filters:
    - "!**/generated/**"
    - "!**/vendor/**"
    - "!**/*.min.js"
    - "!**/migrations/*.sql"

# Debug: add this to your custom instruction and test
- path: "src/test-custom-rules.js"
  instructions: "Always comment 'CUSTOM RULE TEST PASSED' on any change to this file."