Custom Vision Models

Custom Vision terms — defined before code

Choosing the right project type — the most common exam decision

Decision guide — classification vs detection

Labelling differences

The complete Custom Vision pipeline

Precision, Recall, mAP — and the probability threshold tradeoff

Probability threshold — the precision/recall tradeoff

Every prediction has a probability score (0–100%). The probability threshold is your cutoff — below it, the model stays silent. Adjusting it changes the precision/recall balance:

Custom Vision end-to-end — train, publish, and predict

Install: pip install azure-cognitiveservices-vision-customvision

from azure.cognitiveservices.vision.customvision.training import CustomVisionTrainingClient
from azure.cognitiveservices.vision.customvision.training.models import (
    ImageFileCreateBatch,  # Wraps a batch of images with their tags for upload
    ImageFileCreateEntry,  # One image + its tag(s) in a batch
    Region,                # Bounding box for object detection (normalised 0.0–1.0)
)
from azure.cognitiveservices.vision.customvision.prediction import CustomVisionPredictionClient
from msrest.authentication import ApiKeyCredentials
# CustomVision uses msrest credentials, not azure-core credentials
import os
import time  # For polling the training status

# ── Connection details ─────────────────────────────────────────────────
# Custom Vision has SEPARATE endpoints and keys for Training and Prediction
# Find them in customvision.ai → Settings (gear icon)
TRAINING_ENDPOINT = os.environ.get("CUSTOMVISION_TRAINING_ENDPOINT")
TRAINING_KEY      = os.environ.get("CUSTOMVISION_TRAINING_KEY")
PREDICTION_ENDPOINT = os.environ.get("CUSTOMVISION_PREDICTION_ENDPOINT")
PREDICTION_KEY      = os.environ.get("CUSTOMVISION_PREDICTION_KEY")
PREDICTION_RESOURCE_ID = os.environ.get("CUSTOMVISION_PREDICTION_RESOURCE_ID")
# Resource ID format: /subscriptions/xxx/resourceGroups/xxx/providers/...
# Needed when publishing an Iteration — links the model to the prediction resource

# msrest credential wraps the key — different from AzureKeyCredential used elsewhere
training_credentials   = ApiKeyCredentials(in_headers={"Training-key": TRAINING_KEY})
prediction_credentials = ApiKeyCredentials(in_headers={"Prediction-key": PREDICTION_KEY})

# Training client: create projects, upload images, train, evaluate
trainer = CustomVisionTrainingClient(TRAINING_ENDPOINT, training_credentials)
# Prediction client: make predictions against published models
predictor = CustomVisionPredictionClient(PREDICTION_ENDPOINT, prediction_credentials)


# ══════════════════════════════════════════════════════════════════════
# PART 1 — TRAINING: Create project, add tags, upload images, train
# ══════════════════════════════════════════════════════════════════════

# Get the list of available domains — you must use a domain ID, not a name
# Domains: General, Food, Landmarks, Retail, General (compact), etc.
domains = trainer.get_domains()
# Find the General classification domain
general_domain = next(d for d in domains if d.name == "General" and d.type == "Classification")

# Create a new Classification project
# classification_type: "Multiclass" (one tag per image) or "Multilabel" (many tags per image)
project = trainer.create_project(
    name                = "plant-disease-classifier",
    description         = "Classifies plant leaves as Healthy or Diseased",
    domain_id           = general_domain.id,
    classification_type = "Multiclass"   # Each plant is either Healthy OR Diseased
)
project_id = project.id
print(f"Created project: {project.name} (ID: {project_id})")


# ── Create Tags (categories) ───────────────────────────────────────────
# Tags are the categories the model will learn to recognise
# You must create tags BEFORE uploading images
healthy_tag  = trainer.create_tag(project_id, "Healthy")
diseased_tag = trainer.create_tag(project_id, "Diseased")
# Each tag gets a unique ID — use the ID when uploading images
print(f"Created tags: {healthy_tag.name} (ID: {healthy_tag.id}), {diseased_tag.name}")


# ── Upload images with tags ────────────────────────────────────────────
# Build a batch — more efficient than uploading one image at a time
# Batch size limit: 64 images per call
upload_batch = []

healthy_images = ["healthy_leaf_1.jpg", "healthy_leaf_2.jpg", "healthy_leaf_3.jpg"]
for image_path in healthy_images:
    with open(image_path, "rb") as f:
        image_bytes = f.read()  # Read the image as binary
    # ImageFileCreateEntry: wraps one image + its tags for the batch
    # tag_ids: a list of tag IDs — for multiclass, use exactly one tag
    upload_batch.append(ImageFileCreateEntry(
        name    = image_path,
        contents= image_bytes,
        tag_ids = [healthy_tag.id]   # This image belongs to the Healthy category
    ))

diseased_images = ["diseased_leaf_1.jpg", "diseased_leaf_2.jpg", "diseased_leaf_3.jpg"]
for image_path in diseased_images:
    with open(image_path, "rb") as f:
        image_bytes = f.read()
    upload_batch.append(ImageFileCreateEntry(
        name    = image_path,
        contents= image_bytes,
        tag_ids = [diseased_tag.id]  # This image belongs to the Diseased category
    ))

# Upload the entire batch at once — more efficient than one-by-one
upload_result = trainer.create_images_from_files(
    project_id,
    ImageFileCreateBatch(images=upload_batch)  # Wrap the list in a batch object
)

# Check if all images were successfully uploaded
if not upload_result.is_batch_successful:
    # Some images may have failed (unsupported format, duplicate, too small)
    for img in upload_result.images:
        if img.status != "OK":
            print(f"Failed: {img.source_url} — {img.status}")
else:
    print(f"Uploaded {len(upload_batch)} images successfully.")


# ── Train the model ────────────────────────────────────────────────────
# train_project() starts training and returns an Iteration object
# training_type: "Regular" (Quick Training) — for faster/smaller models
# For Advanced Training: training_type="Advanced", budget_in_hours=1 (minimum: 1 hour)
iteration = trainer.train_project(project_id)
print(f"\nTraining started — Iteration: {iteration.id}")

# Training is async — poll until status changes from "Training" to "Completed"
while iteration.status == "Training":
    print("  Still training...")
    time.sleep(10)  # Wait 10 seconds before checking again
    # get_iteration() refreshes the iteration status from the server
    iteration = trainer.get_iteration(project_id, iteration.id)

print(f"Training complete! Status: {iteration.status}")


# ── Evaluate metrics ───────────────────────────────────────────────────
# get_iteration_performance() returns the evaluation metrics for a trained iteration
performance = trainer.get_iteration_performance(
    project_id,
    iteration.id,
    threshold = 0.5   # Probability threshold: only count predictions above 50%
)

# Overall model metrics
print(f"\n=== Model Performance (threshold=50%) ===")
print(f"  Precision: {performance.precision:.1%}")   # :.1% formats 0.923 as "92.3%"
print(f"  Recall:    {performance.recall:.1%}")
print(f"  mAP:       {performance.average_precision:.1%}")  # Mean Average Precision

# Per-tag metrics — see which tags are weak
print("\nPer-tag metrics:")
for tag_perf in performance.per_tag_performance:
    print(f"  {tag_perf.name}: precision={tag_perf.precision:.1%}  recall={tag_perf.recall:.1%}")
    # If Healthy is 95% precision but Diseased is only 60% precision,
    # you need more / better Diseased training images


# ── Publish the Iteration ──────────────────────────────────────────────
# After evaluating, publish the iteration so external apps can use it
# publish_name: a human-readable name for this published model
# prediction_id: the Azure resource ID of your prediction resource
trainer.publish_iteration(
    project_id,
    iteration.id,
    publish_name      = "plant-classifier-v1",
    prediction_id     = PREDICTION_RESOURCE_ID
)
print("\nIteration published as 'plant-classifier-v1'")


# ══════════════════════════════════════════════════════════════════════
# PART 2 — PREDICTION: Call the published model with new images
# ══════════════════════════════════════════════════════════════════════

# ── Classification prediction ──────────────────────────────────────────
with open("new_leaf.jpg", "rb") as f:
    test_image = f.read()

# classify_image(): makes a classification prediction with a local image
# project_id: which project this image belongs to
# published_name: the name you gave when publishing the iteration above
# image_data: the image bytes
results = predictor.classify_image(
    project_id     = project_id,
    published_name = "plant-classifier-v1",  # Must match what you used in publish_iteration()
    image_data     = test_image
)

# results.predictions: list of all tags with their probability scores
# Sorted by probability descending — first item is the most likely prediction
print("\n=== Classification Predictions ===")
for prediction in results.predictions:
    # prediction.tag_name: the label — prediction.probability: confidence (0.0 to 1.0)
    print(f"  {prediction.tag_name}: {prediction.probability:.1%}")

# Best prediction: use the first result (highest probability)
best = results.predictions[0]
if best.probability >= 0.7:       # Our custom threshold: only act if 70%+ confident
    print(f"\nDiagnosis: {best.tag_name} ({best.probability:.1%} confidence)")
else:
    print("\nLow confidence — flag for human review")


# ── URL-based prediction (no local file needed) ─────────────────────────
# classify_image_with_no_store() for predictions that don't store the image
# classify_image_url() for URL-based images
from azure.cognitiveservices.vision.customvision.prediction.models import ImageUrl

url_results = predictor.classify_image_url(
    project_id     = project_id,
    published_name = "plant-classifier-v1",
    image_url      = ImageUrl(url="https://example.com/leaf-photo.jpg")
)
print(f"\nURL prediction: {url_results.predictions[0].tag_name} — {url_results.predictions[0].probability:.1%}")

Python — Object Detection prediction (with bounding boxes)

# Object detection prediction uses detect_image() instead of classify_image()
# Returns both tag names AND bounding box coordinates for each detected object

# Assume you have an object detection project and published iteration
with open("warehouse_photo.jpg", "rb") as f:
    detect_image = f.read()

# detect_image(): same structure as classify_image() but returns bounding boxes
detect_results = predictor.detect_image(
    project_id     = project_id,
    published_name = "defect-detector-v1",
    image_data     = detect_image
)

THRESHOLD = 0.6  # Only report detections above 60% confidence

print("\n=== Object Detection Results ===")
for prediction in detect_results.predictions:
    # Filter out low-confidence predictions using our threshold
    if prediction.probability < THRESHOLD:
        continue   # skip this prediction — not confident enough

    # prediction.bounding_box: NORMALISED coordinates (0.0 to 1.0, not pixels)
    # Multiply by image width/height to get pixel coordinates
    bb = prediction.bounding_box
    # bb.left: x position of left edge as a fraction of image width
    # bb.top:  y position of top edge as a fraction of image height
    # bb.width: width as a fraction of total image width
    # bb.height: height as a fraction of total image height
    print(f"  {prediction.tag_name}: {prediction.probability:.1%}")
    print(f"    Box: left={bb.left:.2f} top={bb.top:.2f} w={bb.width:.2f} h={bb.height:.2f}")
    # Example to convert to pixel coordinates (assuming 1920x1080 image):
    # pixel_x = int(bb.left * 1920)  → left edge in pixels
    # pixel_y = int(bb.top  * 1080)  → top edge in pixels

Custom Vision — what the exam tests

Test your understanding