Master object detection techniques including YOLO, SSD, and Faster R-CNN, understanding anchor boxes, non-maximum suppression, and evaluation metrics like mAP.
Object detection goes beyond image classification by not only identifying what objects are present in an image but also locating them with bounding boxes. This capability is essential for autonomous driving, surveillance, robotics, medical imaging, and countless other applications.
Modern object detection systems must balance accuracy with speed, handle objects of varying sizes, and manage multiple overlapping detections. From the region-based approach of R-CNN to the single-shot efficiency of YOLO, different architectures make different trade-offs to address these challenges.
In this lesson, we'll explore the fundamentals of object detection, understand how anchor boxes and feature pyramids enable multi-scale detection, and learn to evaluate detectors using industry-standard metrics like mean Average Precision (mAP).
Object detection combines classification (what) with localization (where), predicting bounding boxes and class labels for all objects in an image.
Detection vs Classification:
1import torch
2import torch.nn as nn
3import numpy as np
4
5# Classification: Image -> Single class label
6# Detection: Image -> Multiple (box, class, confidence) tuples
7
8class DetectionOutput:
9 """Structure for detection outputs."""
10
11 def __init__(self):
12 self.boxes = [] # [x1, y1, x2, y2] format
13 self.labels = [] # Class indices
14 self.scores = [] # Confidence scores
15
16 def add(self, box, label, score):
17 self.boxes.append(box)
18 self.labels.append(label)
19 self.scores.append(score)
20
21 def to_dict(self):
22 return {
23 'boxes': np.array(self.boxes),
24 'labels': np.array(self.labels),
25 'scores': np.array(self.scores)
26 }
27
28# Example detection output
29detection = DetectionOutput()
30detection.add([100, 50, 200, 150], 0, 0.95) # Person
31detection.add([300, 100, 400, 250], 2, 0.87) # Car
32detection.add([150, 200, 250, 350], 1, 0.72) # Dog
33
34print("Detection results:")
35result = detection.to_dict()
36for i in range(len(result['boxes'])):
37 print(f" Box: {result['boxes'][i]}, "
38 f"Label: {result['labels'][i]}, "
39 f"Score: {result['scores'][i]:.2f}")Bounding Box Representations:
1def convert_box_formats(box, from_format, to_format):
2 """Convert between bounding box formats."""
3
4 # Extract coordinates based on input format
5 if from_format == 'xyxy': # [x1, y1, x2, y2]
6 x1, y1, x2, y2 = box
7 elif from_format == 'xywh': # [x, y, width, height] (top-left)
8 x, y, w, h = box
9 x1, y1, x2, y2 = x, y, x + w, y + h
10 elif from_format == 'cxcywh': # [center_x, center_y, width, height]
11 cx, cy, w, h = box
12 x1, y1, x2, y2 = cx - w/2, cy - h/2, cx + w/2, cy + h/2
13 else:
14 raise ValueError(f"Unknown format: {from_format}")
15
16 # Convert to output format
17 if to_format == 'xyxy':
18 return [x1, y1, x2, y2]
19 elif to_format == 'xywh':
20 return [x1, y1, x2 - x1, y2 - y1]
21 elif to_format == 'cxcywh':
22 return [(x1 + x2) / 2, (y1 + y2) / 2, x2 - x1, y2 - y1]
23 else:
24 raise ValueError(f"Unknown format: {to_format}")
25
26# Test conversions
27box_xyxy = [100, 50, 200, 150]
28box_xywh = convert_box_formats(box_xyxy, 'xyxy', 'xywh')
29box_cxcywh = convert_box_formats(box_xyxy, 'xyxy', 'cxcywh')
30
31print(f"XYXY: {box_xyxy}")
32print(f"XYWH: {box_xywh}")
33print(f"CXCYWH: {box_cxcywh}")Intersection over Union (IoU):
1def compute_iou(box1, box2):
2 """Compute IoU between two boxes in xyxy format."""
3 x1 = max(box1[0], box2[0])
4 y1 = max(box1[1], box2[1])
5 x2 = min(box1[2], box2[2])
6 y2 = min(box1[3], box2[3])
7
8 # Intersection area
9 inter_width = max(0, x2 - x1)
10 inter_height = max(0, y2 - y1)
11 intersection = inter_width * inter_height
12
13 # Union area
14 area1 = (box1[2] - box1[0]) * (box1[3] - box1[1])
15 area2 = (box2[2] - box2[0]) * (box2[3] - box2[1])
16 union = area1 + area2 - intersection
17
18 return intersection / union if union > 0 else 0
19
20# Test IoU
21box_a = [100, 100, 200, 200]
22box_b = [150, 150, 250, 250]
23iou = compute_iou(box_a, box_b)
24print(f"IoU between boxes: {iou:.4f}")Anchor boxes are predefined reference boxes of various sizes and aspect ratios that serve as starting points for detection. Feature pyramids enable detection at multiple scales.
Anchor Box Generation:
1import numpy as np
2import torch
3
4def generate_anchors(feature_size, image_size, scales, ratios):
5 """Generate anchor boxes for a feature map."""
6 anchors = []
7
8 # Calculate stride (how many pixels each feature cell covers)
9 stride = image_size / feature_size
10
11 for y in range(feature_size):
12 for x in range(feature_size):
13 # Center of the anchor in image coordinates
14 cx = (x + 0.5) * stride
15 cy = (y + 0.5) * stride
16
17 for scale in scales:
18 for ratio in ratios:
19 # Width and height
20 w = scale * np.sqrt(ratio) * stride
21 h = scale / np.sqrt(ratio) * stride
22
23 # Convert to xyxy format
24 x1 = cx - w / 2
25 y1 = cy - h / 2
26 x2 = cx + w / 2
27 y2 = cy + h / 2
28
29 anchors.append([x1, y1, x2, y2])
30
31 return np.array(anchors)
32
33# Generate anchors for a 7x7 feature map on 224x224 image
34anchors = generate_anchors(
35 feature_size=7,
36 image_size=224,
37 scales=[0.5, 1.0, 2.0],
38 ratios=[0.5, 1.0, 2.0]
39)
40
41print(f"Total anchors: {len(anchors)}")
42print(f"Anchors per location: {len(anchors) // 49}")
43print(f"First anchor: {anchors[0]}")
44
45# Multi-scale anchors (like SSD)
46class MultiScaleAnchors:
47 """Generate anchors at multiple feature map scales."""
48
49 def __init__(self, image_size=300):
50 self.image_size = image_size
51 # Feature map sizes and corresponding anchor scales
52 self.feature_maps = [38, 19, 10, 5, 3, 1]
53 self.anchor_scales = [0.1, 0.2, 0.375, 0.55, 0.725, 0.9]
54 self.aspect_ratios = [[2], [2, 3], [2, 3], [2, 3], [2], [2]]
55
56 def generate_all(self):
57 all_anchors = []
58
59 for fm_size, scale, ratios in zip(
60 self.feature_maps, self.anchor_scales, self.aspect_ratios
61 ):
62 anchors = self._generate_for_scale(fm_size, scale, ratios)
63 all_anchors.append(anchors)
64 print(f"Feature map {fm_size}x{fm_size}: {len(anchors)} anchors")
65
66 return np.concatenate(all_anchors, axis=0)
67
68 def _generate_for_scale(self, fm_size, scale, extra_ratios):
69 anchors = []
70 ratios = [1.0] + extra_ratios + [1.0/r for r in extra_ratios]
71
72 for y in range(fm_size):
73 for x in range(fm_size):
74 cx = (x + 0.5) / fm_size
75 cy = (y + 0.5) / fm_size
76
77 for ratio in ratios:
78 w = scale * np.sqrt(ratio)
79 h = scale / np.sqrt(ratio)
80 anchors.append([cx, cy, w, h])
81
82 return np.array(anchors)
83
84generator = MultiScaleAnchors()
85all_anchors = generator.generate_all()
86print(f"\nTotal anchors across all scales: {len(all_anchors)}")Feature Pyramid Network (FPN):
1import torch
2import torch.nn as nn
3import torch.nn.functional as F
4
5class FeaturePyramidNetwork(nn.Module):
6 """Feature Pyramid Network for multi-scale feature extraction."""
7
8 def __init__(self, in_channels_list, out_channels=256):
9 super().__init__()
10
11 # Lateral connections (1x1 convs to match channels)
12 self.lateral_convs = nn.ModuleList([
13 nn.Conv2d(in_ch, out_channels, 1)
14 for in_ch in in_channels_list
15 ])
16
17 # Output convolutions (smooth the upsampled features)
18 self.output_convs = nn.ModuleList([
19 nn.Conv2d(out_channels, out_channels, 3, padding=1)
20 for _ in in_channels_list
21 ])
22
23 def forward(self, features):
24 """
25 Args:
26 features: List of feature maps from backbone [C2, C3, C4, C5]
27 Returns:
28 List of FPN features [P2, P3, P4, P5]
29 """
30 # Apply lateral convolutions
31 laterals = [conv(f) for conv, f in zip(self.lateral_convs, features)]
32
33 # Top-down pathway with lateral connections
34 for i in range(len(laterals) - 2, -1, -1):
35 upsampled = F.interpolate(
36 laterals[i + 1],
37 size=laterals[i].shape[-2:],
38 mode='nearest'
39 )
40 laterals[i] = laterals[i] + upsampled
41
42 # Apply output convolutions
43 outputs = [conv(f) for conv, f in zip(self.output_convs, laterals)]
44
45 return outputs
46
47# Example usage
48fpn = FeaturePyramidNetwork(in_channels_list=[256, 512, 1024, 2048])
49# Simulate backbone features at different scales
50c2 = torch.randn(1, 256, 56, 56)
51c3 = torch.randn(1, 512, 28, 28)
52c4 = torch.randn(1, 1024, 14, 14)
53c5 = torch.randn(1, 2048, 7, 7)
54
55pyramid_features = fpn([c2, c3, c4, c5])
56for i, f in enumerate(pyramid_features):
57 print(f"P{i+2} shape: {f.shape}")NMS filters redundant overlapping detections, keeping only the most confident predictions for each object.
Standard NMS Implementation:
1import numpy as np
2import torch
3
4def nms(boxes, scores, iou_threshold=0.5):
5 """Apply Non-Maximum Suppression.
6
7 Args:
8 boxes: numpy array of shape (N, 4) in xyxy format
9 scores: numpy array of shape (N,)
10 iou_threshold: IoU threshold for suppression
11
12 Returns:
13 indices of kept boxes
14 """
15 if len(boxes) == 0:
16 return []
17
18 # Sort by score (descending)
19 order = scores.argsort()[::-1]
20
21 keep = []
22
23 while len(order) > 0:
24 # Keep the highest scoring box
25 i = order[0]
26 keep.append(i)
27
28 if len(order) == 1:
29 break
30
31 # Compute IoU with remaining boxes
32 remaining_boxes = boxes[order[1:]]
33 ious = np.array([compute_iou(boxes[i], box) for box in remaining_boxes])
34
35 # Keep boxes with IoU below threshold
36 mask = ious < iou_threshold
37 order = order[1:][mask]
38
39 return keep
40
41# Test NMS
42test_boxes = np.array([
43 [100, 100, 200, 200], # Box 1
44 [110, 105, 210, 205], # Box 2 (overlaps with 1)
45 [300, 300, 400, 400], # Box 3 (separate)
46 [305, 305, 405, 405], # Box 4 (overlaps with 3)
47])
48test_scores = np.array([0.9, 0.8, 0.85, 0.7])
49
50kept_indices = nms(test_boxes, test_scores, iou_threshold=0.5)
51print(f"Original boxes: {len(test_boxes)}")
52print(f"After NMS: {len(kept_indices)}")
53print(f"Kept indices: {kept_indices}")Soft-NMS:
1def soft_nms(boxes, scores, iou_threshold=0.5, sigma=0.5, score_threshold=0.001):
2 """Soft-NMS: Decay scores instead of hard removal."""
3
4 boxes = boxes.copy()
5 scores = scores.copy()
6 keep = []
7
8 while len(boxes) > 0:
9 # Find max score
10 max_idx = scores.argmax()
11
12 if scores[max_idx] < score_threshold:
13 break
14
15 keep.append(max_idx)
16
17 # Compute IoU with other boxes
18 current_box = boxes[max_idx]
19
20 for i in range(len(boxes)):
21 if i == max_idx:
22 continue
23
24 iou = compute_iou(current_box, boxes[i])
25
26 # Gaussian decay of scores based on IoU
27 scores[i] = scores[i] * np.exp(-(iou ** 2) / sigma)
28
29 # Remove current box
30 boxes = np.delete(boxes, max_idx, axis=0)
31 scores = np.delete(scores, max_idx, axis=0)
32
33 return keep
34
35# PyTorch batched NMS
36def batched_nms(boxes, scores, labels, iou_threshold=0.5):
37 """NMS with class-specific suppression."""
38
39 # Offset boxes by class to prevent cross-class suppression
40 max_coord = boxes.max()
41 offsets = labels.float() * (max_coord + 1)
42 boxes_offset = boxes + offsets[:, None]
43
44 # Apply standard NMS
45 keep = torch.ops.torchvision.nms(boxes_offset, scores, iou_threshold)
46
47 return keepDifferent detection architectures make different trade-offs between accuracy and speed.
YOLO (You Only Look Once):
1import torch
2import torch.nn as nn
3
4class YOLOHead(nn.Module):
5 """Simplified YOLO detection head."""
6
7 def __init__(self, in_channels, num_classes, num_anchors=3):
8 super().__init__()
9 self.num_classes = num_classes
10 self.num_anchors = num_anchors
11
12 # Output: anchors * (4 box coords + 1 objectness + num_classes)
13 out_channels = num_anchors * (5 + num_classes)
14
15 self.conv = nn.Sequential(
16 nn.Conv2d(in_channels, in_channels * 2, 3, padding=1),
17 nn.BatchNorm2d(in_channels * 2),
18 nn.LeakyReLU(0.1),
19 nn.Conv2d(in_channels * 2, out_channels, 1)
20 )
21
22 def forward(self, x):
23 batch_size = x.size(0)
24 output = self.conv(x)
25
26 # Reshape: [B, A*(5+C), H, W] -> [B, A, H, W, 5+C]
27 output = output.view(
28 batch_size,
29 self.num_anchors,
30 5 + self.num_classes,
31 output.size(2),
32 output.size(3)
33 ).permute(0, 1, 3, 4, 2)
34
35 return output
36
37# Using pretrained YOLOv5 with ultralytics
38def load_yolo():
39 """Load pretrained YOLO model."""
40 # pip install ultralytics
41 # from ultralytics import YOLO
42 # model = YOLO('yolov8n.pt')
43 # results = model('image.jpg')
44 passFaster R-CNN:
1import torch
2from torchvision.models.detection import fasterrcnn_resnet50_fpn_v2
3from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
4
5def create_faster_rcnn(num_classes):
6 """Create Faster R-CNN with custom number of classes."""
7
8 # Load pretrained model
9 model = fasterrcnn_resnet50_fpn_v2(weights='DEFAULT')
10
11 # Replace classifier head
12 in_features = model.roi_heads.box_predictor.cls_score.in_features
13 model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
14
15 return model
16
17# Usage
18model = create_faster_rcnn(num_classes=10)
19model.eval()
20
21# Inference
22x = torch.randn(1, 3, 800, 800)
23with torch.no_grad():
24 predictions = model(x)
25
26print("Faster R-CNN output:")
27print(f" Boxes shape: {predictions[0]['boxes'].shape}")
28print(f" Labels shape: {predictions[0]['labels'].shape}")
29print(f" Scores shape: {predictions[0]['scores'].shape}")Architecture Comparison:
1def compare_architectures():
2 """Compare different detection architectures."""
3
4 print("Detection Architecture Comparison")
5 print("=" * 60)
6 print(f"{'Architecture':<15} {'Speed':<10} {'Accuracy':<10} {'Use Case':<25}")
7 print("-" * 60)
8
9 architectures = [
10 ("YOLOv8", "Very Fast", "Good", "Real-time, edge devices"),
11 ("SSD", "Fast", "Good", "Mobile, embedded systems"),
12 ("Faster R-CNN", "Slow", "Excellent", "High accuracy needed"),
13 ("RetinaNet", "Medium", "Very Good", "Dense object detection"),
14 ("DETR", "Medium", "Very Good", "End-to-end detection"),
15 ]
16
17 for arch in architectures:
18 print(f"{arch[0]:<15} {arch[1]:<10} {arch[2]:<10} {arch[3]:<25}")
19
20compare_architectures()Build a complete object detection system with preprocessing, inference, NMS, and visualization.
1"""
2Complete Object Detection Pipeline
3End-to-end detection with pretrained models.
4"""
5
6import torch
7import torch.nn as nn
8import numpy as np
9from torchvision import transforms
10from torchvision.models.detection import fasterrcnn_resnet50_fpn_v2
11
12# COCO class names
13COCO_CLASSES = [
14 '__background__', 'person', 'bicycle', 'car', 'motorcycle', 'airplane',
15 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant',
16 'N/A', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog',
17 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'N/A',
18 'backpack', 'umbrella', 'N/A', 'N/A', 'handbag', 'tie', 'suitcase',
19 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat',
20 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle',
21 'N/A', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana',
22 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza',
23 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'N/A',
24 'dining table', 'N/A', 'N/A', 'toilet', 'N/A', 'tv', 'laptop', 'mouse',
25 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster',
26 'sink', 'refrigerator', 'N/A', 'book', 'clock', 'vase', 'scissors',
27 'teddy bear', 'hair drier', 'toothbrush'
28]
29
30class ObjectDetector:
31 """Complete object detection pipeline."""
32
33 def __init__(self, model_name='faster_rcnn', confidence_threshold=0.5):
34 self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
35 self.confidence_threshold = confidence_threshold
36 self.model = self._load_model(model_name)
37 self.class_names = COCO_CLASSES
38
39 def _load_model(self, model_name):
40 """Load pretrained detection model."""
41 if model_name == 'faster_rcnn':
42 model = fasterrcnn_resnet50_fpn_v2(weights='DEFAULT')
43 else:
44 raise ValueError(f"Unknown model: {model_name}")
45
46 model = model.to(self.device)
47 model.eval()
48 return model
49
50 def preprocess(self, image):
51 """Preprocess image for detection."""
52 if isinstance(image, np.ndarray):
53 # Convert BGR to RGB if needed
54 if len(image.shape) == 3 and image.shape[2] == 3:
55 image = image[:, :, ::-1].copy()
56
57 # Convert to tensor
58 image = torch.from_numpy(image).permute(2, 0, 1).float() / 255.0
59
60 return image.to(self.device)
61
62 def postprocess(self, predictions, image_shape):
63 """Filter and format predictions."""
64 boxes = predictions['boxes'].cpu().numpy()
65 labels = predictions['labels'].cpu().numpy()
66 scores = predictions['scores'].cpu().numpy()
67
68 # Filter by confidence
69 mask = scores > self.confidence_threshold
70 boxes = boxes[mask]
71 labels = labels[mask]
72 scores = scores[mask]
73
74 # Convert to detection format
75 detections = []
76 for box, label, score in zip(boxes, labels, scores):
77 detections.append({
78 'box': box.tolist(),
79 'class_id': int(label),
80 'class_name': self.class_names[label],
81 'confidence': float(score)
82 })
83
84 return detections
85
86 @torch.no_grad()
87 def detect(self, image):
88 """Run detection on an image."""
89 # Preprocess
90 input_tensor = self.preprocess(image)
91
92 # Inference
93 predictions = self.model([input_tensor])[0]
94
95 # Postprocess
96 detections = self.postprocess(predictions, image.shape[:2])
97
98 return detections
99
100 def detect_batch(self, images):
101 """Run detection on multiple images."""
102 tensors = [self.preprocess(img) for img in images]
103
104 with torch.no_grad():
105 predictions = self.model(tensors)
106
107 results = []
108 for pred, img in zip(predictions, images):
109 results.append(self.postprocess(pred, img.shape[:2]))
110
111 return results
112
113 def summarize(self, detections):
114 """Summarize detection results."""
115 from collections import Counter
116
117 class_counts = Counter(d['class_name'] for d in detections)
118
119 print(f"\nDetection Summary:")
120 print(f"Total objects: {len(detections)}")
121 print(f"\nObjects by class:")
122 for class_name, count in class_counts.most_common():
123 print(f" {class_name}: {count}")
124
125
126# Example usage
127print("Creating object detector...")
128detector = ObjectDetector(confidence_threshold=0.5)
129
130# Create test image
131test_image = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8)
132
133# Run detection
134print("Running detection...")
135detections = detector.detect(test_image)
136
137print(f"\nFound {len(detections)} objects")
138for det in detections[:5]:
139 print(f" {det['class_name']}: {det['confidence']:.2f} at {det['box']}")
140
141detector.summarize(detections)This complete pipeline demonstrates production-ready object detection including model loading, preprocessing, inference, confidence filtering, and result summarization. The modular design supports easy extension for custom models and post-processing requirements.
In the next lesson, we'll explore semantic segmentation, learning how to classify every pixel in an image to create detailed scene understanding maps.
Your progress is saved in this browser, on this device only.