Ray Job Submission Tutorial¶
Run a CheckMAITE capability asynchronously with the registry-backed Ray job backend. This notebook has explicit local and remote/shared-cluster modes so local-only paths are never presented as safe remote defaults.
Who this is for: ML Engineer running analyses for a team on shared compute, or a Data Scientist whose analyses outgrow their local machine.
Where this fits: stage 5, Run at scale, of the T&E workflow. It submits a stage 2–4 capability to a local or shared Ray cluster as a tracked, reattachable job. Start with the Ray Simple Job Submission tutorial for single-driver work.
Configuration¶
Set CHECKMAITE_RAY_MODE=remote for a shared cluster. Remote mode requires:
CHECKMAITE_RAY_ADDRESS: Ray Client or cluster address;CHECKMAITE_RAY_SCOPE: stable workspace/user scope used after notebook restarts;CHECKMAITE_TUTORIAL_DATA_ROOT: worker-visible CheckMAITE COCO fixture directory;CHECKMAITE_TUTORIAL_ANALYTICS_URI: shared filesystem or object-store prefix;CHECKMAITE_TUTORIAL_ARTIFACT_URI: durable prefix used to externalize oversized inline reports.
Local mode writes to CHECKMAITE_TUTORIAL_OUTPUT_ROOT (default: ./checkmaite-ray-tutorial-output) so results survive backend shutdown; remove that directory explicitly when it is no longer needed. It may reconnect to a fresh local runtime. Remote mode never uses force_reinit=True because disconnecting an active shared client can disrupt in-flight work.
import os
import platform
from pathlib import Path
import ray
from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
import checkmaite
from checkmaite.core.analytics_store import AnalyticsStore, ParquetBackend
from checkmaite.core.object_detection import DataevalCleaning
from checkmaite.core.object_detection.dataset_loaders import load_coco_detection_dataset
from checkmaite.jobs import (
JobStatus,
configure_job_backend,
get_job,
list_jobs,
shutdown_job_backend,
submit_capability,
)
mode = os.getenv("CHECKMAITE_RAY_MODE", "local").strip().lower()
if mode not in {"local", "remote"}:
raise ValueError("CHECKMAITE_RAY_MODE must be 'local' or 'remote'")
if mode == "remote":
ray_address = os.environ["CHECKMAITE_RAY_ADDRESS"]
idempotency_scope = os.environ["CHECKMAITE_RAY_SCOPE"]
dataset_root = Path(os.environ["CHECKMAITE_TUTORIAL_DATA_ROOT"]).expanduser()
analytics_uri = os.environ["CHECKMAITE_TUTORIAL_ANALYTICS_URI"]
artifact_uri = os.environ["CHECKMAITE_TUTORIAL_ARTIFACT_URI"]
force_reinit = False
else:
ray_address = "local"
idempotency_scope = "checkmaite-ray-tutorial-local"
candidates = [Path.cwd(), *Path.cwd().parents]
repo_root = next(
(
path
for path in candidates
if (path / "pyproject.toml").is_file()
and (path / "src/checkmaite").is_dir()
and (path / "tests/data_for_tests/coco_dataset/ann_file.json").is_file()
),
None,
)
if repo_root is None:
raise RuntimeError("Run local mode from the CheckMAITE source tree")
dataset_root = repo_root / "tests/data_for_tests/coco_dataset"
tutorial_root = (
Path(os.getenv("CHECKMAITE_TUTORIAL_OUTPUT_ROOT", "./checkmaite-ray-tutorial-output")).expanduser().resolve()
)
tutorial_root.mkdir(parents=True, exist_ok=True)
analytics_uri = str(tutorial_root / "analytics")
artifact_uri = str(tutorial_root / "report-artifacts")
force_reinit = True
dataset_ann = dataset_root / "ann_file.json"
if not dataset_root.is_dir() or not dataset_ann.is_file():
raise FileNotFoundError(f"Expected CheckMAITE COCO fixture at {dataset_root}")
analytics_store_config = {
"backend": "parquet",
"uri": analytics_uri,
}
artifact_store_config = {"uri": artifact_uri}
print(f"Mode: {mode}; scope: {idempotency_scope}")
print(f"Analytics: {analytics_uri}")
print(f"Report artifacts: {artifact_uri}")
Connect and validate the cluster¶
The version probe runs on a Ray worker, not in the notebook process. Resource checks happen before submission so the tutorial does not silently create an unschedulable workload.
@ray.remote(num_cpus=0)
def worker_environment(dataset_path: str) -> dict[str, str | bool]:
import platform
from pathlib import Path
import ray
import checkmaite
dataset = Path(dataset_path)
return {
"python": platform.python_version(),
"ray": ray.__version__,
"checkmaite": checkmaite.__version__,
"dataset_ready": dataset.is_dir() and (dataset / "ann_file.json").is_file(),
}
Submit, wait safely, inspect, and clean up¶
A bounded wait() may return a non-terminal status. Choose whether to cancel or continue waiting before calling result(). shutdown_job_backend() disconnects this client; it does not promise immediate removal of detached registry/controller actors, which follow their configured retention policies.
job = None
try:
configure_job_backend(
"ray",
address=ray_address,
force_reinit=force_reinit,
idempotency_scope=idempotency_scope,
analytics_store=analytics_store_config,
artifact_store=artifact_store_config,
)
client_environment = {
"python": platform.python_version(),
"ray": ray.__version__,
"checkmaite": checkmaite.__version__,
}
node_ids = [node["NodeID"] for node in ray.nodes() if node["Alive"]]
worker_refs = [
worker_environment.options(
scheduling_strategy=NodeAffinitySchedulingStrategy(node_id=node_id, soft=False)
).remote(str(dataset_root))
for node_id in node_ids
]
workers = ray.get(worker_refs, timeout=30)
for node_id, worker in zip(node_ids, workers, strict=True):
worker_versions = {key: worker[key] for key in client_environment}
if worker_versions != client_environment:
raise RuntimeError(
f"Client/worker version mismatch on {node_id}: client={client_environment}, worker={worker_versions}"
)
if not worker["dataset_ready"]:
raise RuntimeError(f"Dataset is not visible to Ray node {node_id} at {dataset_root}")
required_cpus = 1.0
cluster = ray.cluster_resources()
available = ray.available_resources()
if cluster.get("CPU", 0.0) < required_cpus:
raise RuntimeError(f"Cluster has {cluster.get('CPU', 0.0)} CPUs; this tutorial requires {required_cpus}")
if available.get("CPU", 0.0) < required_cpus:
print("No CPU is currently free; the job may remain SCHEDULING until shared capacity is released")
print(f"Cluster resources: {cluster}")
print(f"Currently available resources: {available}")
dataset = load_coco_detection_dataset(
root=str(dataset_root),
ann_file=str(dataset_ann),
dataset_id="coco-job-tutorial",
)
capability = DataevalCleaning()
job = submit_capability(capability, datasets=[dataset], use_cache=False)
print(f"Submitted {job.job_id}: {job.status}")
final_status = job.wait(timeout=120)
if not final_status.is_terminal:
action = input("Job is still active. Enter 'cancel' or 'wait': ").strip().lower()
if action == "cancel":
job.cancel()
final_status = job.wait(timeout=30)
if not final_status.is_terminal:
raise RuntimeError("Cancellation is still pending; result() was not requested")
elif action == "wait":
final_status = job.wait(timeout=None)
else:
raise RuntimeError("Result was not requested because the job is still active")
if final_status is not JobStatus.COMPLETED:
print(f"Job ended with {final_status}; result() will raise the corresponding job error")
ref = job.result(timeout=10)
print(f"Run UID: {ref.run_uid}")
print(f"Store URI: {ref.store_uri}")
print(f"Report: {ref.report}")
for tracked_job in list_jobs(limit=10):
print(tracked_job.job_id, tracked_job.status, tracked_job.job_name)
fetched = get_job(job.job_id)
print("Fetched after submission:", fetched.job_id, fetched.status)
store = AnalyticsStore(ParquetBackend(analytics_uri))
print(f"Analytics tables: {store.list_tables()}")
finally:
# Drop the API object without an unbounded wait, then disconnect this driver.
# Detached remote actors continue according to cluster retention policy.
shutdown_job_backend(wait=False)
if ray.is_initialized():
ray.shutdown()
print("Disconnected this notebook's Ray client; detached actor retention is cluster-managed.")