Understand the architecture and mechanics of CNNs, from convolution operations and pooling layers to modern architectures like VGG, ResNet, and EfficientNet.
Convolutional Neural Networks (CNNs) revolutionized computer vision by learning hierarchical feature representations directly from images. Unlike traditional methods that rely on handcrafted features, CNNs automatically discover optimal filters through backpropagation, achieving superhuman performance on many visual tasks.
The breakthrough came in 2012 when AlexNet won the ImageNet competition by a significant margin, demonstrating that deep CNNs could learn powerful visual representations. Since then, architectures have evolved from simple stacked convolutions to sophisticated designs with skip connections, dense connections, and neural architecture search.
In this lesson, we'll build a deep understanding of CNN fundamentals, explore how convolution and pooling operations work, and study the architectural innovations that have driven progress in computer vision.
The convolution operation is the core building block of CNNs. It applies learned filters to input images to detect patterns at various locations.
Understanding Convolution:
1import torch
2import torch.nn as nn
3import numpy as np
4
5def manual_convolution(image, kernel):
6 """Implement 2D convolution manually."""
7 img_h, img_w = image.shape
8 ker_h, ker_w = kernel.shape
9
10 # Output dimensions (no padding, stride=1)
11 out_h = img_h - ker_h + 1
12 out_w = img_w - ker_w + 1
13
14 output = np.zeros((out_h, out_w))
15
16 for i in range(out_h):
17 for j in range(out_w):
18 # Extract patch
19 patch = image[i:i+ker_h, j:j+ker_w]
20 # Element-wise multiply and sum
21 output[i, j] = np.sum(patch * kernel)
22
23 return output
24
25# Example: Edge detection kernel
26edge_kernel = np.array([
27 [-1, -1, -1],
28 [-1, 8, -1],
29 [-1, -1, -1]
30])
31
32# Test image
33test_image = np.random.rand(10, 10)
34result = manual_convolution(test_image, edge_kernel)
35print(f"Input: {test_image.shape} -> Output: {result.shape}")PyTorch Conv2d Layer:
1import torch
2import torch.nn as nn
3
4# Create a convolutional layer
5conv = nn.Conv2d(
6 in_channels=3, # RGB input
7 out_channels=64, # Number of filters
8 kernel_size=3, # 3x3 kernels
9 stride=1, # Move 1 pixel at a time
10 padding=1, # Same padding
11 bias=True # Add learnable bias
12)
13
14# Input: [batch, channels, height, width]
15x = torch.randn(1, 3, 224, 224)
16output = conv(x)
17
18print(f"Input shape: {x.shape}")
19print(f"Output shape: {output.shape}")
20print(f"Weight shape: {conv.weight.shape}") # [out_ch, in_ch, ker_h, ker_w]
21print(f"Parameters: {sum(p.numel() for p in conv.parameters())}")
22
23# Output size calculation
24def calc_output_size(input_size, kernel_size, stride, padding):
25 """Calculate output size of convolution."""
26 return (input_size - kernel_size + 2 * padding) // stride + 1
27
28h_out = calc_output_size(224, 3, 1, 1)
29print(f"Calculated output height: {h_out}")Different Convolution Types:
1# Standard convolution
2standard_conv = nn.Conv2d(64, 128, kernel_size=3, padding=1)
3
4# Depthwise separable convolution (used in MobileNet)
5class DepthwiseSeparableConv(nn.Module):
6 def __init__(self, in_channels, out_channels):
7 super().__init__()
8 # Depthwise: separate filter per input channel
9 self.depthwise = nn.Conv2d(
10 in_channels, in_channels,
11 kernel_size=3, padding=1, groups=in_channels
12 )
13 # Pointwise: 1x1 conv to mix channels
14 self.pointwise = nn.Conv2d(in_channels, out_channels, kernel_size=1)
15
16 def forward(self, x):
17 x = self.depthwise(x)
18 x = self.pointwise(x)
19 return x
20
21# Compare parameters
22standard = nn.Conv2d(64, 128, 3, padding=1)
23depthwise_sep = DepthwiseSeparableConv(64, 128)
24
25standard_params = sum(p.numel() for p in standard.parameters())
26sep_params = sum(p.numel() for p in depthwise_sep.parameters())
27
28print(f"Standard conv params: {standard_params:,}")
29print(f"Depthwise separable params: {sep_params:,}")
30print(f"Reduction: {standard_params / sep_params:.1f}x")
31
32# Dilated convolution (increases receptive field)
33dilated_conv = nn.Conv2d(64, 64, kernel_size=3, padding=2, dilation=2)Pooling layers reduce spatial dimensions while preserving important features, providing translation invariance and computational efficiency.
Pooling Operations:
1import torch
2import torch.nn as nn
3
4# Max pooling - takes maximum value in each window
5max_pool = nn.MaxPool2d(kernel_size=2, stride=2)
6
7# Average pooling - takes average value
8avg_pool = nn.AvgPool2d(kernel_size=2, stride=2)
9
10# Global average pooling - reduces to single value per channel
11global_avg_pool = nn.AdaptiveAvgPool2d(output_size=1)
12
13# Test
14x = torch.randn(1, 64, 32, 32)
15
16print(f"Input: {x.shape}")
17print(f"After MaxPool2d(2): {max_pool(x).shape}")
18print(f"After AvgPool2d(2): {avg_pool(x).shape}")
19print(f"After GlobalAvgPool: {global_avg_pool(x).shape}")
20
21# Manual max pooling
22def manual_max_pool(x, pool_size=2):
23 """Implement max pooling manually."""
24 b, c, h, w = x.shape
25 h_out, w_out = h // pool_size, w // pool_size
26
27 output = torch.zeros(b, c, h_out, w_out)
28
29 for i in range(h_out):
30 for j in range(w_out):
31 h_start, h_end = i * pool_size, (i + 1) * pool_size
32 w_start, w_end = j * pool_size, (j + 1) * pool_size
33
34 window = x[:, :, h_start:h_end, w_start:w_end]
35 output[:, :, i, j] = window.max(dim=-1)[0].max(dim=-1)[0]
36
37 return output
38
39# Strided convolution as alternative to pooling
40strided_conv = nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1)
41print(f"After strided conv: {strided_conv(x).shape}")Pooling Effects:
1def demonstrate_pooling_effects():
2 """Show how pooling provides translation invariance."""
3
4 # Create feature map with a peak
5 feature_map = torch.zeros(1, 1, 8, 8)
6 feature_map[0, 0, 2, 2] = 1.0 # Peak at position (2, 2)
7
8 # Shift the peak
9 shifted_map = torch.zeros(1, 1, 8, 8)
10 shifted_map[0, 0, 3, 3] = 1.0 # Peak at position (3, 3)
11
12 max_pool = nn.MaxPool2d(4, 4)
13
14 pooled_original = max_pool(feature_map)
15 pooled_shifted = max_pool(shifted_map)
16
17 print("Original peak at (2,2):")
18 print(pooled_original.squeeze())
19 print("\nShifted peak at (3,3):")
20 print(pooled_shifted.squeeze())
21 print("\nBoth produce same pooled output (translation invariance)")
22
23demonstrate_pooling_effects()A complete CNN architecture combines convolutional blocks, pooling, and fully connected layers into a unified feature extraction and classification system.
Basic CNN Block:
1import torch
2import torch.nn as nn
3
4class ConvBlock(nn.Module):
5 """Standard convolutional block: Conv -> BatchNorm -> ReLU"""
6
7 def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1):
8 super().__init__()
9 self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding)
10 self.bn = nn.BatchNorm2d(out_channels)
11 self.relu = nn.ReLU(inplace=True)
12
13 def forward(self, x):
14 return self.relu(self.bn(self.conv(x)))
15
16# Simple CNN architecture
17class SimpleCNN(nn.Module):
18 """Basic CNN for image classification."""
19
20 def __init__(self, num_classes=10):
21 super().__init__()
22
23 # Feature extraction
24 self.features = nn.Sequential(
25 # Block 1: 3 -> 64
26 ConvBlock(3, 64),
27 ConvBlock(64, 64),
28 nn.MaxPool2d(2, 2),
29
30 # Block 2: 64 -> 128
31 ConvBlock(64, 128),
32 ConvBlock(128, 128),
33 nn.MaxPool2d(2, 2),
34
35 # Block 3: 128 -> 256
36 ConvBlock(128, 256),
37 ConvBlock(256, 256),
38 nn.MaxPool2d(2, 2),
39 )
40
41 # Global average pooling
42 self.avgpool = nn.AdaptiveAvgPool2d(1)
43
44 # Classifier
45 self.classifier = nn.Sequential(
46 nn.Flatten(),
47 nn.Linear(256, 128),
48 nn.ReLU(inplace=True),
49 nn.Dropout(0.5),
50 nn.Linear(128, num_classes)
51 )
52
53 def forward(self, x):
54 x = self.features(x)
55 x = self.avgpool(x)
56 x = self.classifier(x)
57 return x
58
59# Test
60model = SimpleCNN(num_classes=10)
61x = torch.randn(1, 3, 224, 224)
62output = model(x)
63
64print(f"Input: {x.shape}")
65print(f"Output: {output.shape}")
66print(f"Total parameters: {sum(p.numel() for p in model.parameters()):,}")Understanding Receptive Field:
1def calculate_receptive_field(layers):
2 """Calculate receptive field of a CNN."""
3 rf = 1 # Initial receptive field
4 stride = 1 # Cumulative stride
5
6 for name, kernel_size, layer_stride in layers:
7 rf = rf + (kernel_size - 1) * stride
8 stride = stride * layer_stride
9 print(f"{name}: RF = {rf}, stride = {stride}")
10
11 return rf
12
13# Example: VGG-like architecture
14layers = [
15 ("conv3_1", 3, 1),
16 ("conv3_2", 3, 1),
17 ("pool", 2, 2),
18 ("conv3_3", 3, 1),
19 ("conv3_4", 3, 1),
20 ("pool", 2, 2),
21]
22
23final_rf = calculate_receptive_field(layers)
24print(f"\nFinal receptive field: {final_rf}x{final_rf} pixels")Understanding key CNN architectures helps in selecting and designing models for specific tasks.
VGG Architecture:
1import torch
2import torch.nn as nn
3
4class VGGBlock(nn.Module):
5 """VGG-style block with repeated 3x3 convolutions."""
6
7 def __init__(self, in_channels, out_channels, num_convs):
8 super().__init__()
9 layers = []
10 for i in range(num_convs):
11 layers.append(nn.Conv2d(
12 in_channels if i == 0 else out_channels,
13 out_channels, kernel_size=3, padding=1
14 ))
15 layers.append(nn.ReLU(inplace=True))
16 layers.append(nn.MaxPool2d(2, 2))
17 self.block = nn.Sequential(*layers)
18
19 def forward(self, x):
20 return self.block(x)
21
22class VGG16(nn.Module):
23 """VGG16 architecture."""
24
25 def __init__(self, num_classes=1000):
26 super().__init__()
27 self.features = nn.Sequential(
28 VGGBlock(3, 64, 2), # 224 -> 112
29 VGGBlock(64, 128, 2), # 112 -> 56
30 VGGBlock(128, 256, 3), # 56 -> 28
31 VGGBlock(256, 512, 3), # 28 -> 14
32 VGGBlock(512, 512, 3), # 14 -> 7
33 )
34 self.classifier = nn.Sequential(
35 nn.Flatten(),
36 nn.Linear(512 * 7 * 7, 4096),
37 nn.ReLU(inplace=True),
38 nn.Dropout(0.5),
39 nn.Linear(4096, 4096),
40 nn.ReLU(inplace=True),
41 nn.Dropout(0.5),
42 nn.Linear(4096, num_classes),
43 )
44
45 def forward(self, x):
46 x = self.features(x)
47 x = self.classifier(x)
48 return xResNet with Skip Connections:
1class ResidualBlock(nn.Module):
2 """Basic residual block with skip connection."""
3
4 def __init__(self, in_channels, out_channels, stride=1):
5 super().__init__()
6
7 self.conv1 = nn.Conv2d(in_channels, out_channels, 3, stride, 1, bias=False)
8 self.bn1 = nn.BatchNorm2d(out_channels)
9 self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, 1, bias=False)
10 self.bn2 = nn.BatchNorm2d(out_channels)
11 self.relu = nn.ReLU(inplace=True)
12
13 # Skip connection
14 self.skip = nn.Identity()
15 if stride != 1 or in_channels != out_channels:
16 self.skip = nn.Sequential(
17 nn.Conv2d(in_channels, out_channels, 1, stride, bias=False),
18 nn.BatchNorm2d(out_channels)
19 )
20
21 def forward(self, x):
22 identity = self.skip(x)
23
24 out = self.conv1(x)
25 out = self.bn1(out)
26 out = self.relu(out)
27
28 out = self.conv2(out)
29 out = self.bn2(out)
30
31 out += identity # Skip connection
32 out = self.relu(out)
33
34 return out
35
36class ResNet18(nn.Module):
37 """ResNet-18 architecture."""
38
39 def __init__(self, num_classes=1000):
40 super().__init__()
41
42 # Initial conv
43 self.conv1 = nn.Sequential(
44 nn.Conv2d(3, 64, 7, 2, 3, bias=False),
45 nn.BatchNorm2d(64),
46 nn.ReLU(inplace=True),
47 nn.MaxPool2d(3, 2, 1)
48 )
49
50 # Residual layers
51 self.layer1 = self._make_layer(64, 64, 2, stride=1)
52 self.layer2 = self._make_layer(64, 128, 2, stride=2)
53 self.layer3 = self._make_layer(128, 256, 2, stride=2)
54 self.layer4 = self._make_layer(256, 512, 2, stride=2)
55
56 # Classifier
57 self.avgpool = nn.AdaptiveAvgPool2d(1)
58 self.fc = nn.Linear(512, num_classes)
59
60 def _make_layer(self, in_channels, out_channels, num_blocks, stride):
61 layers = [ResidualBlock(in_channels, out_channels, stride)]
62 for _ in range(1, num_blocks):
63 layers.append(ResidualBlock(out_channels, out_channels))
64 return nn.Sequential(*layers)
65
66 def forward(self, x):
67 x = self.conv1(x)
68 x = self.layer1(x)
69 x = self.layer2(x)
70 x = self.layer3(x)
71 x = self.layer4(x)
72 x = self.avgpool(x)
73 x = x.view(x.size(0), -1)
74 x = self.fc(x)
75 return x
76
77# Compare architectures
78vgg = VGG16(num_classes=1000)
79resnet = ResNet18(num_classes=1000)
80
81print(f"VGG16 parameters: {sum(p.numel() for p in vgg.parameters()):,}")
82print(f"ResNet18 parameters: {sum(p.numel() for p in resnet.parameters()):,}")Modern Architectures - EfficientNet:
1# Using torchvision's pretrained models
2from torchvision import models
3
4# Load EfficientNet
5efficientnet = models.efficientnet_b0(weights='IMAGENET1K_V1')
6
7# Architecture components
8print("EfficientNet-B0 structure:")
9print(f"Parameters: {sum(p.numel() for p in efficientnet.parameters()):,}")
10
11# EfficientNet uses:
12# - Mobile Inverted Bottleneck (MBConv)
13# - Squeeze-and-Excitation blocks
14# - Compound scaling (width, depth, resolution)
15
16# Test inference
17x = torch.randn(1, 3, 224, 224)
18with torch.no_grad():
19 out = efficientnet(x)
20print(f"Output shape: {out.shape}")Implement a complete CNN architecture with modern components including residual connections, batch normalization, and dropout.
1"""
2Custom CNN Implementation
3Build a modern CNN with best practices for image classification.
4"""
5
6import torch
7import torch.nn as nn
8import torch.nn.functional as F
9
10class SEBlock(nn.Module):
11 """Squeeze-and-Excitation block for channel attention."""
12
13 def __init__(self, channels, reduction=16):
14 super().__init__()
15 self.fc1 = nn.Linear(channels, channels // reduction)
16 self.fc2 = nn.Linear(channels // reduction, channels)
17
18 def forward(self, x):
19 b, c, _, _ = x.size()
20
21 # Squeeze: global average pooling
22 y = x.view(b, c, -1).mean(dim=2)
23
24 # Excitation: FC -> ReLU -> FC -> Sigmoid
25 y = F.relu(self.fc1(y))
26 y = torch.sigmoid(self.fc2(y))
27
28 # Scale
29 y = y.view(b, c, 1, 1)
30 return x * y
31
32
33class ConvBNReLU(nn.Module):
34 """Convolution + BatchNorm + ReLU."""
35
36 def __init__(self, in_ch, out_ch, kernel_size=3, stride=1, padding=1):
37 super().__init__()
38 self.conv = nn.Conv2d(in_ch, out_ch, kernel_size, stride, padding, bias=False)
39 self.bn = nn.BatchNorm2d(out_ch)
40 self.relu = nn.ReLU(inplace=True)
41
42 def forward(self, x):
43 return self.relu(self.bn(self.conv(x)))
44
45
46class ResidualBlock(nn.Module):
47 """Residual block with SE attention."""
48
49 def __init__(self, in_ch, out_ch, stride=1, use_se=True):
50 super().__init__()
51
52 self.conv1 = ConvBNReLU(in_ch, out_ch, stride=stride)
53 self.conv2 = nn.Sequential(
54 nn.Conv2d(out_ch, out_ch, 3, 1, 1, bias=False),
55 nn.BatchNorm2d(out_ch)
56 )
57
58 self.se = SEBlock(out_ch) if use_se else nn.Identity()
59
60 self.shortcut = nn.Identity()
61 if stride != 1 or in_ch != out_ch:
62 self.shortcut = nn.Sequential(
63 nn.Conv2d(in_ch, out_ch, 1, stride, bias=False),
64 nn.BatchNorm2d(out_ch)
65 )
66
67 self.relu = nn.ReLU(inplace=True)
68
69 def forward(self, x):
70 identity = self.shortcut(x)
71
72 out = self.conv1(x)
73 out = self.conv2(out)
74 out = self.se(out)
75
76 out += identity
77 out = self.relu(out)
78
79 return out
80
81
82class ModernCNN(nn.Module):
83 """Modern CNN with residual connections and attention."""
84
85 def __init__(self, num_classes=10, channels=[64, 128, 256, 512]):
86 super().__init__()
87
88 # Stem
89 self.stem = nn.Sequential(
90 ConvBNReLU(3, channels[0], kernel_size=7, stride=2, padding=3),
91 nn.MaxPool2d(3, 2, 1)
92 )
93
94 # Stages
95 self.stage1 = self._make_stage(channels[0], channels[0], num_blocks=2)
96 self.stage2 = self._make_stage(channels[0], channels[1], num_blocks=2, stride=2)
97 self.stage3 = self._make_stage(channels[1], channels[2], num_blocks=2, stride=2)
98 self.stage4 = self._make_stage(channels[2], channels[3], num_blocks=2, stride=2)
99
100 # Head
101 self.avgpool = nn.AdaptiveAvgPool2d(1)
102 self.dropout = nn.Dropout(0.2)
103 self.fc = nn.Linear(channels[3], num_classes)
104
105 # Initialize weights
106 self._init_weights()
107
108 def _make_stage(self, in_ch, out_ch, num_blocks, stride=1):
109 layers = [ResidualBlock(in_ch, out_ch, stride)]
110 for _ in range(1, num_blocks):
111 layers.append(ResidualBlock(out_ch, out_ch))
112 return nn.Sequential(*layers)
113
114 def _init_weights(self):
115 for m in self.modules():
116 if isinstance(m, nn.Conv2d):
117 nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
118 elif isinstance(m, nn.BatchNorm2d):
119 nn.init.constant_(m.weight, 1)
120 nn.init.constant_(m.bias, 0)
121 elif isinstance(m, nn.Linear):
122 nn.init.normal_(m.weight, 0, 0.01)
123 nn.init.constant_(m.bias, 0)
124
125 def forward(self, x):
126 # Feature extraction
127 x = self.stem(x)
128 x = self.stage1(x)
129 x = self.stage2(x)
130 x = self.stage3(x)
131 x = self.stage4(x)
132
133 # Classification
134 x = self.avgpool(x)
135 x = x.view(x.size(0), -1)
136 x = self.dropout(x)
137 x = self.fc(x)
138
139 return x
140
141 def get_features(self, x):
142 """Extract features before classification."""
143 x = self.stem(x)
144 x = self.stage1(x)
145 x = self.stage2(x)
146 x = self.stage3(x)
147 x = self.stage4(x)
148 x = self.avgpool(x)
149 return x.view(x.size(0), -1)
150
151
152# Test the model
153model = ModernCNN(num_classes=10)
154x = torch.randn(2, 3, 224, 224)
155
156# Forward pass
157output = model(x)
158print(f"Input shape: {x.shape}")
159print(f"Output shape: {output.shape}")
160
161# Model statistics
162total_params = sum(p.numel() for p in model.parameters())
163trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
164
165print(f"\nModel Statistics:")
166print(f"Total parameters: {total_params:,}")
167print(f"Trainable parameters: {trainable_params:,}")
168print(f"Model size: {total_params * 4 / 1024 / 1024:.2f} MB")
169
170# Feature extraction
171features = model.get_features(x)
172print(f"\nFeature vector shape: {features.shape}")
173
174# Layer-by-layer analysis
175print("\nLayer-by-layer output shapes:")
176with torch.no_grad():
177 test_x = torch.randn(1, 3, 224, 224)
178 print(f"Input: {test_x.shape}")
179
180 test_x = model.stem(test_x)
181 print(f"After stem: {test_x.shape}")
182
183 test_x = model.stage1(test_x)
184 print(f"After stage1: {test_x.shape}")
185
186 test_x = model.stage2(test_x)
187 print(f"After stage2: {test_x.shape}")
188
189 test_x = model.stage3(test_x)
190 print(f"After stage3: {test_x.shape}")
191
192 test_x = model.stage4(test_x)
193 print(f"After stage4: {test_x.shape}")This implementation demonstrates modern CNN design principles including residual connections for gradient flow, Squeeze-and-Excitation attention for channel recalibration, proper weight initialization, and dropout regularization. The architecture follows best practices while remaining simple enough to understand and modify.
In the next lesson, we'll apply CNNs to image classification tasks, learning about data augmentation, transfer learning, and fine-tuning pretrained models for custom datasets.
Your progress is saved in this browser, on this device only.