Feature Flag Testing Best Practices: Control Risk in Production

import pytest
from unittest.mock import patch
from myapp.checkout import CheckoutService

@pytest.mark.parametrize("new_payment_flow_enabled", [True, False])
def test_checkout_completes_with_flag(new_payment_flow_enabled):
    """Checkout should complete regardless of new_payment_flow flag state."""
    with patch_feature_flag("new_payment_flow", new_payment_flow_enabled):
        service = CheckoutService()
        result = service.process_order(
            user_id="user_123",
            cart_id="cart_456",
            payment_method="card_789"
        )
        
        assert result.success
        assert result.order_id is not None

@pytest.mark.parametrize("new_payment_flow_enabled", [True, False])
def test_checkout_fails_gracefully_on_payment_error(new_payment_flow_enabled):
    """Payment errors should be handled correctly in both flag states."""
    with patch_feature_flag("new_payment_flow", new_payment_flow_enabled):
        with patch_payment_provider(should_fail=True):
            service = CheckoutService()
            result = service.process_order(user_id="user_123", cart_id="cart_456")
            
            assert not result.success
            assert result.error_code == "payment_failed"

from contextlib import contextmanager

@contextmanager
def feature_flags(**flags: bool):
    """Context manager for controlling feature flags in tests."""
    with unittest.mock.patch.object(
        FeatureFlagService, 
        'is_enabled',
        side_effect=lambda name, **kwargs: flags.get(name, False)
    ):
        yield

def test_new_checkout_shows_updated_ui():
    with feature_flags(new_checkout_flow=True):
        response = client.get("/checkout")
        assert "new-checkout-container" in response.text

def test_legacy_checkout_shows_old_ui():
    with feature_flags(new_checkout_flow=False):
        response = client.get("/checkout")
        assert "legacy-checkout-container" in response.text

CHECKOUT_FLAGS = {
    "new_checkout_flow": [True, False],
    "saved_payment_methods": [True, False],
    "express_checkout": [True, False],
}

def get_critical_flag_combinations():
    """Return only the combinations that matter for correctness."""
    flags = list(CHECKOUT_FLAGS.keys())
    
    yield {f: False for f in flags}  # All off (baseline)
    yield {f: True for f in flags}   # All on
    
    for flag in flags:
        # Each flag in isolation
        yield {f: (f == flag) for f in flags}

@pytest.mark.parametrize("flag_config", list(get_critical_flag_combinations()))
def test_checkout_core_behavior(flag_config):
    """Core checkout behavior must work for all critical flag combinations."""
    with feature_flags(**flag_config):
        result = complete_test_checkout()
        assert result.success, f"Checkout failed with flags: {flag_config}"

def test_flag_enabled_for_beta_users():
    """new_dashboard flag should be enabled for beta users."""
    beta_user = User(id="beta_001", groups=["beta"])
    regular_user = User(id="user_002", groups=[])
    
    assert is_flag_enabled("new_dashboard", user=beta_user)
    assert not is_flag_enabled("new_dashboard", user=regular_user)

def test_percentage_rollout_within_bounds():
    """5% rollout should enable for approximately 5% of users."""
    sample_size = 10000
    enabled_count = sum(
        1 for i in range(sample_size)
        if is_flag_enabled("new_feature", user=User(id=f"user_{i}"))
    )
    
    percentage = enabled_count / sample_size
    assert 0.04 <= percentage <= 0.06, \
        f"5% rollout enabled for {percentage:.1%} (expected ~5%)"

def test_rollout_is_deterministic():
    """Same user should consistently get same flag state."""
    user = User(id="user_12345")
    results = [is_flag_enabled("gradual_rollout", user=user) for _ in range(10)]
    assert len(set(results)) == 1, "Flag evaluation is not deterministic for same user"

def test_no_long_lived_feature_flags():
    """Feature flags older than 90 days should be removed."""
    all_flags = flag_service.list_flags()
    old_flags = [
        f for f in all_flags 
        if f.created_at < datetime.now() - timedelta(days=90)
        and not f.is_permanent
    ]
    
    assert len(old_flags) == 0, \
        f"Old flags needing cleanup: {[f.name for f in old_flags]}"

# .github/workflows/feature-flag-tests.yml
name: Feature Flag Tests

jobs:
  test-flag-states:
    strategy:
      matrix:
        flag-config:
          - name: "all-flags-off"
            env: "NEW_CHECKOUT=false,NEW_DASHBOARD=false"
          - name: "all-flags-on"
            env: "NEW_CHECKOUT=true,NEW_DASHBOARD=true"
          - name: "checkout-only"
            env: "NEW_CHECKOUT=true,NEW_DASHBOARD=false"
    
    steps:
      - uses: actions/checkout@v4
      - name: Run tests
        run: pytest tests/
        env:
          FEATURE_FLAGS: ${{ matrix.flag-config.env }}

def track_conversion_by_flag(user_id: str, flag_name: str, event: str):
    flag_state = "enabled" if is_flag_enabled(flag_name, user_id=user_id) else "disabled"
    
    metrics.increment(
        "user_event",
        tags={
            "event": event,
            f"flag_{flag_name}": flag_state
        }
    )

# Compare conversion rates between flag-enabled and flag-disabled users
track_conversion_by_flag(user_id, "new_checkout_flow", "checkout_completed")
track_conversion_by_flag(user_id, "new_checkout_flow", "checkout_abandoned")

from scipy import stats

def should_increase_rollout(flag_name: str, metric: str, days: int = 7) -> bool:
    """Returns True if the flag's metric improvement is statistically significant."""
    enabled_data = get_metric_data(flag=flag_name, state="enabled", days=days)
    disabled_data = get_metric_data(flag=flag_name, state="disabled", days=days)
    
    t_stat, p_value = stats.ttest_ind(enabled_data, disabled_data)
    
    enabled_mean = sum(enabled_data) / len(enabled_data)
    disabled_mean = sum(disabled_data) / len(disabled_data)
    relative_lift = (enabled_mean - disabled_mean) / disabled_mean
    
    print(f"Flag: {flag_name}, Metric: {metric}")
    print(f"Enabled mean: {enabled_mean:.4f}, Disabled mean: {disabled_mean:.4f}")
    print(f"Relative lift: {relative_lift:.2%}, p-value: {p_value:.4f}")
    
    return p_value < 0.05 and relative_lift > 0

# Before increasing rollout from 10% to 50%
if should_increase_rollout("new_checkout", "conversion_rate"):
    increase_rollout_to_50_percent()
else:
    print("Not enough evidence to increase rollout")