Remote Inference with Ray Serve¶
Deploy one small CheckMAITE model safely to either a dedicated local Ray runtime or a shared cluster. Application and deployment names are unique, resources are checked before deployment, and cleanup deletes only the application created by this notebook.
Who this is for: ML Engineer serving a model on shared compute, or a Data Scientist evaluating against a model that is already served.
Where this fits: stage 5, Run at scale, of the T&E workflow. It serves a stage 1 model with Ray Serve and runs a stage 3 evaluation against the remote deployment.
Shared-cluster prerequisites¶
For shared mode set:
CHECKMAITE_RAY_MODE=remote;CHECKMAITE_RAY_ADDRESSto the documented Ray Client/cluster address;CHECKMAITE_RAY_DASHBOARD_URLto a browser-accessible dashboard URL;CHECKMAITE_RAY_MODEL_FILEto an immutable, pre-provisionedcenternet-resnet18.pthvisible from Serve workers;CHECKMAITE_RAY_MODEL_SHA256to that file's full SHA-256 digest;CHECKMAITE_VISDRONE_DATA_ROOTto the client-visible tutorial dataset.
The client, Serve workers, and cluster image must use compatible Python, Ray, and CheckMAITE versions. Shared mode never downloads weights from a replica: a worker verifies the pre-provisioned file before deployment. Local mode is explicit and intended only for a disposable development machine.
import os
import platform
import uuid
from pathlib import Path
import ray
from ray import serve
import checkmaite
mode = os.getenv("CHECKMAITE_RAY_MODE", "local").strip().lower()
if mode not in {"local", "remote"}:
raise ValueError("CHECKMAITE_RAY_MODE must be 'local' or 'remote'")
if mode == "remote":
address = os.environ["CHECKMAITE_RAY_ADDRESS"]
dashboard_url = os.environ["CHECKMAITE_RAY_DASHBOARD_URL"]
model_file = Path(os.environ["CHECKMAITE_RAY_MODEL_FILE"]).expanduser()
model_sha256 = os.environ["CHECKMAITE_RAY_MODEL_SHA256"].strip().lower()
if model_file.suffix != ".pth" or len(model_sha256) != 64:
raise ValueError("Remote model file must be .pth and CHECKMAITE_RAY_MODEL_SHA256 must be a full digest")
model_dir = str(model_file.parent)
model_name = model_file.stem
dataset_root = Path(os.environ["CHECKMAITE_VISDRONE_DATA_ROOT"]).expanduser()
else:
address = "local"
dashboard_url = ray_address = None
candidates = [Path.cwd(), *Path.cwd().parents]
repo_root = next(
(
path
for path in candidates
if (path / "pyproject.toml").is_file()
and (path / "src/checkmaite").is_dir()
and (path / "tests/data_for_tests/visdrone_dataset").is_dir()
),
None,
)
if repo_root is None:
raise RuntimeError("Run local mode from the CheckMAITE source tree")
model_dir = str(repo_root / ".checkmaite-models")
model_name = None
model_sha256 = None
dataset_root = repo_root / "tests/data_for_tests/visdrone_dataset"
if not dataset_root.is_dir():
raise FileNotFoundError(f"Expected the VisDrone tutorial dataset at {dataset_root}")
Validate versions and capacity¶
This tutorial deploys only one CPU replica by default. Increase resources only after checking cluster capacity and coordinating with other users.
@ray.remote(num_cpus=0)
def worker_environment(model_path: str, expected_sha256: str | None) -> dict[str, str | bool]:
import hashlib
import platform
import uuid
from pathlib import Path
import ray
import checkmaite
path = Path(model_path)
try:
if expected_sha256 is not None:
digest = hashlib.sha256(path.read_bytes()).hexdigest()
model_path_ready = digest == expected_sha256
else:
path.mkdir(parents=True, exist_ok=True)
probe = path / f".checkmaite-write-probe-{uuid.uuid4().hex}"
probe.write_bytes(b"probe")
probe.unlink()
model_path_ready = True
except OSError:
model_path_ready = False
return {
"python": platform.python_version(),
"ray": ray.__version__,
"checkmaite": checkmaite.__version__,
"model_path_ready": model_path_ready,
}
Deploy, evaluate, and clean up only this application¶
The application name includes a unique suffix so it cannot replace another user's deployment. serve.delete() runs in finally; this notebook never calls cluster-wide serve.shutdown().
import hashlib
import shutil
from typing import Any
from checkmaite.core.object_detection import MaiteEvaluation
from checkmaite.core.object_detection.dataset_loaders import load_visdrone_detection_dataset
from checkmaite.core.object_detection.metrics import map50_torch_metric_factory
from checkmaite.core.object_detection.models import VisdroneODModel
from checkmaite.core.serving.rayserve import RayObjectDetectionClient, RayObjectDetectionDeployment, print_serve_status
suffix = f"{os.getenv('USER', 'user')}-{uuid.uuid4().hex}"
app_name = f"checkmaite-tutorial-{suffix}"
deployment_name = f"visdrone-{suffix}"
created = False
connected = False
local_model_run_dir = None
class VerifiedVisdroneODModel(VisdroneODModel):
"""VisDrone wrapper that verifies pre-provisioned weights inside the Serve replica."""
def __init__(self, *, expected_sha256: str, **kwargs: Any) -> None:
model_path = Path(kwargs["model_pickle_dir"]) / f"{kwargs['model_name']}.pth"
actual_sha256 = hashlib.sha256(model_path.read_bytes()).hexdigest()
if actual_sha256 != expected_sha256:
raise RuntimeError(f"Model integrity check failed inside Serve replica: {model_path}")
super().__init__(**kwargs)
try:
ray.init(address=address)
connected = True
if dashboard_url:
print(f"Ray dashboard: {dashboard_url}")
client_environment = {
"python": platform.python_version(),
"ray": ray.__version__,
"checkmaite": checkmaite.__version__,
}
probe_path = str(model_file) if mode == "remote" else model_dir
worker = ray.get(worker_environment.remote(probe_path, model_sha256), timeout=30)
worker_versions = {key: worker[key] for key in client_environment}
if worker_versions != client_environment:
raise RuntimeError(f"Client/worker version mismatch: client={client_environment}, worker={worker_versions}")
if not worker["model_path_ready"]:
raise RuntimeError(f"Model source is unavailable or failed integrity validation: {probe_path}")
required_cpus = 1.0
cluster = ray.cluster_resources()
available = ray.available_resources()
if cluster.get("CPU", 0.0) < required_cpus:
raise RuntimeError(f"Cluster has {cluster.get('CPU', 0.0)} CPUs; deployment requires {required_cpus}")
if available.get("CPU", 0.0) < required_cpus:
raise RuntimeError("No CPU is currently available; retry after shared-cluster capacity is released")
print(f"Cluster resources: {cluster}")
print(f"Currently available resources: {available}")
model_kwargs = {"arch": "resnet18", "model_pickle_dir": model_dir, "batch_size": 1}
model_class = VisdroneODModel
if model_name is not None:
model_class = VerifiedVisdroneODModel
model_kwargs["model_name"] = model_name
model_kwargs["expected_sha256"] = model_sha256
else:
# Isolate the disposable local-mode download from other notebook runs.
local_model_run_dir = str(Path(model_dir) / suffix)
model_kwargs["model_pickle_dir"] = local_model_run_dir
deployment = RayObjectDetectionDeployment.options(
name=deployment_name,
num_replicas=1,
ray_actor_options={"num_cpus": required_cpus, "num_gpus": 0},
).bind(model_class, model_kwargs)
created = True
handle = serve.run(deployment, name=app_name, route_prefix=None)
print_serve_status()
client = RayObjectDetectionClient(handle)
dataset = load_visdrone_detection_dataset(root=str(dataset_root))
metric = map50_torch_metric_factory()
metric.metadata["id"] = metric.return_key
result = MaiteEvaluation().run(
use_cache=False,
datasets=[dataset],
metrics=[metric],
models=[client],
)
print(result.outputs)
finally:
try:
if created:
serve.delete(app_name, _blocking=True)
print(f"Deleted only tutorial application {app_name!r}")
finally:
if connected:
ray.shutdown()
if local_model_run_dir is not None:
shutil.rmtree(local_model_run_dir, ignore_errors=True)
Operational notes¶
Use the externally reachable dashboard URL supplied by your platform rather than 127.0.0.1. On a shared cluster, never call serve.shutdown() or ray stop: those are cluster-wide operations that can affect unrelated users and applications.
Common problems¶
- Model weight downloads fail with
CERTIFICATE_VERIFY_FAILEDon a cluster behind a TLS-inspecting proxy. Mount the organization CA bundle into the Ray pods and setSSL_CERT_FILE. See Worker environments. (#711)