Testing ML Model Serving Endpoints: Latency, Drift, and Schema Validation
Model serving is where machine learning meets production reality. A model that scores well in a notebook can fail silently in production — returning predictions with wrong shapes, degraded accuracy on shifted data, or latency that breaks SLAs. This guide covers systematic testing for ML model serving endpoints across TensorFlow Serving, NVIDIA Triton, and BentoML.
Why Serving Endpoint Testing Is Different
Standard API testing validates that a server returns expected responses. Model serving testing adds three harder problems:
- Non-determinism — stochastic models return different outputs on repeated calls, making exact-match assertions useless.
- Distribution shift — a model can return valid-shaped responses while accuracy degrades because input data drifted.
- Latency tails — p99 latency matters more than mean; a slow tail breaks real-time applications.
Your test suite needs to handle all three.
Schema Validation for Model APIs
Before testing accuracy or latency, validate that the endpoint accepts and returns the right shape. Schema mismatches cause silent failures that are worse than crashes — they produce garbage predictions with a 200 OK.
TensorFlow Serving
TF Serving exposes a gRPC API and a REST API. Test the REST shape first:
import pytest
import requests
import numpy as np
TFSERVING_URL = "http://localhost:8501/v1/models/my_model:predict"
def make_request(inputs: list) -> requests.Response:
payload = {"instances": inputs}
return requests.post(TFSERVING_URL, json=payload, timeout=5)
class TestSchemaValidation:
def test_valid_input_returns_200(self):
inputs = [[1.0, 2.0, 3.0, 4.0]] # shape [1, 4]
resp = make_request(inputs)
assert resp.status_code == 200
def test_response_has_predictions_key(self):
inputs = [[1.0, 2.0, 3.0, 4.0]]
resp = make_request(inputs)
body = resp.json()
assert "predictions" in body
def test_output_shape_matches_expected(self):
batch_size = 4
inputs = [[float(i)] * 4 for i in range(batch_size)]
resp = make_request(inputs)
predictions = resp.json()["predictions"]
assert len(predictions) == batch_size
def test_wrong_feature_count_returns_error(self):
# Model expects 4 features, send 3
inputs = [[1.0, 2.0, 3.0]]
resp = make_request(inputs)
assert resp.status_code in (400, 500)
def test_empty_batch_returns_error(self):
resp = make_request([])
assert resp.status_code in (400, 500)
def test_string_input_rejected(self):
inputs = [["a", "b", "c", "d"]]
resp = make_request(inputs)
assert resp.status_code in (400, 500)Triton Inference Server
Triton uses a more structured HTTP protocol. Validate the model metadata endpoint before testing predictions:
import tritonclient.http as httpclient
import numpy as np
import pytest
TRITON_URL = "localhost:8000"
MODEL_NAME = "resnet50"
MODEL_VERSION = "1"
@pytest.fixture(scope="module")
def triton_client():
client = httpclient.InferenceServerClient(url=TRITON_URL)
assert client.is_server_live(), "Triton server is not live"
assert client.is_server_ready(), "Triton server is not ready"
return client
class TestTritonSchema:
def test_model_is_ready(self, triton_client):
assert triton_client.is_model_ready(MODEL_NAME, MODEL_VERSION)
def test_model_metadata_shape(self, triton_client):
metadata = triton_client.get_model_metadata(MODEL_NAME, MODEL_VERSION)
inputs = {inp["name"]: inp for inp in metadata["inputs"]}
outputs = {out["name"]: out for out in metadata["outputs"]}
# Validate input tensor exists and has correct dtype
assert "input__0" in inputs
assert inputs["input__0"]["datatype"] == "FP32"
assert inputs["input__0"]["shape"] == [-1, 3, 224, 224] # NCHW
# Validate output tensor
assert "output__0" in outputs
def test_inference_output_shape(self, triton_client):
input_data = np.random.rand(1, 3, 224, 224).astype(np.float32)
inputs = [httpclient.InferInput("input__0", input_data.shape, "FP32")]
inputs[0].set_data_from_numpy(input_data)
outputs = [httpclient.InferRequestedOutput("output__0")]
result = triton_client.infer(MODEL_NAME, inputs, outputs=outputs)
output = result.as_numpy("output__0")
assert output.shape == (1, 1000) # ImageNet classes
assert output.dtype == np.float32Using Pydantic for Contract Testing
Define contracts in code so they're versioned alongside models:
from pydantic import BaseModel, validator, Field
from typing import List
import numpy as np
class PredictionRequest(BaseModel):
instances: List[List[float]]
@validator("instances")
def validate_feature_count(cls, v):
expected_features = 4
for row in v:
if len(row) != expected_features:
raise ValueError(f"Expected {expected_features} features, got {len(row)}")
return v
class PredictionResponse(BaseModel):
predictions: List[List[float]]
model_version: str = Field(default=None)
@validator("predictions")
def validate_probability_sum(cls, v):
for pred in v:
total = sum(pred)
if not (0.99 <= total <= 1.01):
raise ValueError(f"Probabilities must sum to 1.0, got {total}")
return v
def test_response_satisfies_contract():
resp = requests.post(TFSERVING_URL, json={"instances": [[1.0, 2.0, 3.0, 4.0]]})
body = resp.json()
# Will raise ValidationError if schema is wrong
PredictionResponse(**body)Latency Testing
Latency SLAs are useless without percentile testing. Mean latency hides tail problems.
import time
import statistics
import concurrent.futures
from dataclasses import dataclass
from typing import List
@dataclass
class LatencyResult:
p50: float
p95: float
p99: float
p999: float
errors: int
total: int
def measure_latency(
url: str,
payload: dict,
n_requests: int = 200,
concurrency: int = 10,
) -> LatencyResult:
latencies = []
errors = 0
def single_request():
start = time.perf_counter()
try:
resp = requests.post(url, json=payload, timeout=10)
resp.raise_for_status()
return time.perf_counter() - start, None
except Exception as e:
return None, str(e)
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
futures = [executor.submit(single_request) for _ in range(n_requests)]
for future in concurrent.futures.as_completed(futures):
latency, error = future.result()
if error:
errors += 1
else:
latencies.append(latency * 1000) # convert to ms
latencies.sort()
n = len(latencies)
return LatencyResult(
p50=latencies[int(n * 0.50)],
p95=latencies[int(n * 0.95)],
p99=latencies[int(n * 0.99)],
p999=latencies[int(n * 0.999)] if n >= 1000 else latencies[-1],
errors=errors,
total=n_requests,
)
class TestLatencySLAs:
PAYLOAD = {"instances": [[1.0, 2.0, 3.0, 4.0]]}
def test_p50_under_10ms(self):
result = measure_latency(TFSERVING_URL, self.PAYLOAD, n_requests=100)
assert result.p50 < 10, f"p50 latency {result.p50:.1f}ms exceeds 10ms SLA"
def test_p99_under_100ms(self):
result = measure_latency(TFSERVING_URL, self.PAYLOAD, n_requests=200)
assert result.p99 < 100, f"p99 latency {result.p99:.1f}ms exceeds 100ms SLA"
def test_error_rate_under_1_percent(self):
result = measure_latency(TFSERVING_URL, self.PAYLOAD, n_requests=200)
error_rate = result.errors / result.total
assert error_rate < 0.01, f"Error rate {error_rate:.1%} exceeds 1%"
def test_batch_latency_scales_sublinearly(self):
single = measure_latency(TFSERVING_URL, {"instances": [[1.0] * 4]}, n_requests=50)
batch = measure_latency(
TFSERVING_URL,
{"instances": [[1.0] * 4] * 32},
n_requests=50,
)
# Batch of 32 should not take 32x as long
ratio = batch.p50 / single.p50
assert ratio < 10, f"Batch scaling ratio {ratio:.1f}x is too high"Accuracy Drift Detection
A model returning valid schema but degraded accuracy is the silent killer. Test with a golden dataset — a fixed set of inputs with known expected outputs.
import json
from pathlib import Path
import numpy as np
GOLDEN_DATASET_PATH = Path("tests/fixtures/golden_dataset.json")
def load_golden_dataset():
with open(GOLDEN_DATASET_PATH) as f:
data = json.load(f)
return data["inputs"], data["expected_labels"]
class TestAccuracyThresholds:
def test_accuracy_above_baseline(self):
inputs, expected_labels = load_golden_dataset()
payload = {"instances": inputs}
resp = requests.post(TFSERVING_URL, json=payload)
predictions = resp.json()["predictions"]
predicted_labels = [np.argmax(p) for p in predictions]
correct = sum(p == e for p, e in zip(predicted_labels, expected_labels))
accuracy = correct / len(expected_labels)
ACCURACY_BASELINE = 0.92 # Set from last validated model version
assert accuracy >= ACCURACY_BASELINE, (
f"Model accuracy {accuracy:.3f} dropped below baseline {ACCURACY_BASELINE}"
)
def test_confidence_distribution_reasonable(self):
inputs, _ = load_golden_dataset()
payload = {"instances": inputs[:50]}
resp = requests.post(TFSERVING_URL, json=payload)
predictions = resp.json()["predictions"]
max_confidences = [max(p) for p in predictions]
mean_confidence = np.mean(max_confidences)
# A model that's always uncertain (<50% max confidence) is broken
assert mean_confidence > 0.5, (
f"Mean max confidence {mean_confidence:.3f} is too low — model may be degraded"
)
def test_no_nan_in_predictions(self):
inputs, _ = load_golden_dataset()
resp = requests.post(TFSERVING_URL, json={"instances": inputs})
predictions = resp.json()["predictions"]
for i, pred in enumerate(predictions):
assert not any(np.isnan(v) for v in pred), (
f"NaN found in prediction {i}: {pred}"
)
def test_class_distribution_not_degenerate(self):
"""Model should not always predict the same class."""
inputs, _ = load_golden_dataset()
resp = requests.post(TFSERVING_URL, json={"instances": inputs})
predictions = resp.json()["predictions"]
predicted_labels = [np.argmax(p) for p in predictions]
unique_classes = len(set(predicted_labels))
assert unique_classes > 1, "Model is predicting only one class — likely degenerate"Health Check Testing
Health checks for serving infrastructure need to cover more than HTTP 200.
class TestHealthChecks:
def test_tfserving_model_status_endpoint(self):
resp = requests.get(
"http://localhost:8501/v1/models/my_model",
timeout=5,
)
assert resp.status_code == 200
status = resp.json()
assert "model_version_status" in status
versions = status["model_version_status"]
assert len(versions) > 0
# At least one version should be AVAILABLE
states = [v["state"] for v in versions]
assert "AVAILABLE" in states, f"No available versions, states: {states}"
def test_bentoml_readiness(self):
resp = requests.get("http://localhost:3000/readyz", timeout=5)
assert resp.status_code == 200
def test_bentoml_liveness(self):
resp = requests.get("http://localhost:3000/livez", timeout=5)
assert resp.status_code == 200
def test_triton_health_live(self, triton_client):
assert triton_client.is_server_live()
def test_triton_health_ready(self, triton_client):
assert triton_client.is_server_ready()
def test_model_reload_after_version_update(self):
"""Verify that a new model version becomes available without restart."""
# Deploy new version (out of scope for this test — assume it happened)
import time
time.sleep(2) # Poll window for TF Serving auto-reload
resp = requests.get("http://localhost:8501/v1/models/my_model")
status = resp.json()
versions = status["model_version_status"]
version_numbers = [v["version"] for v in versions if v["state"] == "AVAILABLE"]
# Should have at least the version we expect
assert max(int(v) for v in version_numbers) >= 2Integration Testing with BentoML
BentoML adds a service layer. Test the full stack including preprocessing:
import bentoml
from bentoml.testing.server import run_bento_server
import pytest
@pytest.fixture(scope="module")
def bento_server():
with run_bento_server("my_classifier:latest", host="127.0.0.1", port=5000) as server:
yield server
class TestBentoMLIntegration:
def test_classify_endpoint_returns_label(self, bento_server):
resp = requests.post(
f"http://{bento_server.host}:{bento_server.port}/classify",
json={"features": [1.0, 2.0, 3.0, 4.0]},
)
assert resp.status_code == 200
body = resp.json()
assert "label" in body
assert isinstance(body["label"], str)
def test_batch_endpoint_handles_large_payload(self, bento_server):
batch = [{"features": [float(i)] * 4} for i in range(256)]
resp = requests.post(
f"http://{bento_server.host}:{bento_server.port}/classify_batch",
json=batch,
timeout=30,
)
assert resp.status_code == 200
results = resp.json()
assert len(results) == 256CI Integration
Wire latency and schema tests into your deployment pipeline:
# .github/workflows/model-serving-tests.yaml
name: Model Serving Tests
on:
push:
paths:
- "models/**"
- "serving/**"
jobs:
serving-tests:
runs-on: ubuntu-latest
services:
tfserving:
image: tensorflow/serving:latest
ports:
- 8501:8501
env:
MODEL_NAME: my_model
volumes:
- ./models:/models/my_model
steps:
- uses: actions/checkout@v4
- name: Run schema validation tests
run: pytest tests/serving/test_schema.py -v
- name: Run latency SLA tests
run: pytest tests/serving/test_latency.py -v --timeout=120
- name: Run accuracy baseline tests
run: pytest tests/serving/test_accuracy.py -vSummary
Testing ML model serving requires a different mindset than API testing. The key areas:
- Schema validation: Use Pydantic contracts, test edge cases (empty batch, wrong dtype, wrong shape)
- Latency: Test p95/p99, not mean. Test under realistic concurrency.
- Accuracy: Maintain a golden dataset. Fail the build if accuracy drops below baseline.
- Health checks: Test model availability state, not just HTTP liveness.
- Integration: Test the full stack including preprocessing and postprocessing.
For end-to-end testing of serving endpoints in staging environments — including writing tests in plain English and running them on a schedule — HelpMeTest handles the orchestration so you can focus on the ML-specific assertions.