Master encoder-decoder architectures for machine translation, text summarization, and other sequence transformation tasks.
Sequence-to-sequence (seq2seq) models transform one sequence into another, powering applications like machine translation, text summarization, question answering, and chatbots. The architecture consists of an encoder that processes the input sequence and a decoder that generates the output sequence.
The breakthrough came with attention mechanisms, allowing the decoder to focus on relevant parts of the input at each generation step. This addressed the information bottleneck in fixed-length encodings and dramatically improved performance on longer sequences.
In this lesson, you'll build seq2seq models from scratch and understand the attention mechanism that revolutionized NLP.
The basic seq2seq framework:
Encoder: Processes input sequence into a context representation
Decoder: Generates output sequence from context, token by token
1class Encoder:
2 def __init__(self, vocab_size, embed_dim, hidden_dim):
3 self.embedding = Embedding(vocab_size, embed_dim)
4 self.rnn = LSTM(embed_dim, hidden_dim)
5
6 def forward(self, x):
7 # x: (batch, seq_len)
8 embedded = self.embedding(x) # (batch, seq_len, embed_dim)
9 outputs, (hidden, cell) = self.rnn(embedded)
10 return outputs, hidden, cell
11
12class Decoder:
13 def __init__(self, vocab_size, embed_dim, hidden_dim):
14 self.embedding = Embedding(vocab_size, embed_dim)
15 self.rnn = LSTM(embed_dim, hidden_dim)
16 self.output = Linear(hidden_dim, vocab_size)
17
18 def forward(self, x, hidden, cell):
19 embedded = self.embedding(x)
20 output, (hidden, cell) = self.rnn(embedded, (hidden, cell))
21 prediction = self.output(output)
22 return prediction, hidden, cellAttention allows the decoder to focus on different parts of the input.
1def attention(query, keys, values):
2 """
3 query: decoder hidden state (batch, hidden_dim)
4 keys: encoder outputs (batch, src_len, hidden_dim)
5 values: encoder outputs (batch, src_len, hidden_dim)
6 """
7 # Compute attention scores
8 scores = torch.bmm(keys, query.unsqueeze(2)) # (batch, src_len, 1)
9 scores = scores.squeeze(2) # (batch, src_len)
10
11 # Softmax to get attention weights
12 weights = F.softmax(scores, dim=1) # (batch, src_len)
13
14 # Weighted sum of values
15 context = torch.bmm(weights.unsqueeze(1), values) # (batch, 1, hidden)
16 context = context.squeeze(1) # (batch, hidden)
17
18 return context, weights
19
20# In decoder:
21context, attn_weights = attention(hidden, encoder_outputs, encoder_outputs)
22combined = torch.cat([context, embedded], dim=1)
23output = self.rnn(combined, hidden)Beam search explores multiple hypotheses for better generation.
1def beam_search(model, src, beam_width=5, max_len=50):
2 """Beam search decoding"""
3 encoder_outputs, hidden = model.encoder(src)
4
5 # Initialize beams: (score, sequence, hidden_state)
6 beams = [(0, [BOS_TOKEN], hidden)]
7
8 for _ in range(max_len):
9 all_candidates = []
10
11 for score, seq, hidden in beams:
12 if seq[-1] == EOS_TOKEN:
13 all_candidates.append((score, seq, hidden))
14 continue
15
16 # Get next token probabilities
17 last_token = torch.tensor([seq[-1]])
18 logits, new_hidden = model.decoder(last_token, hidden, encoder_outputs)
19 log_probs = F.log_softmax(logits, dim=-1)
20
21 # Expand beam
22 top_probs, top_indices = log_probs.topk(beam_width)
23 for prob, idx in zip(top_probs[0], top_indices[0]):
24 new_score = score + prob.item()
25 new_seq = seq + [idx.item()]
26 all_candidates.append((new_score, new_seq, new_hidden))
27
28 # Keep top beams
29 beams = sorted(all_candidates, key=lambda x: x[0], reverse=True)[:beam_width]
30
31 # Stop if all beams ended
32 if all(b[1][-1] == EOS_TOKEN for b in beams):
33 break
34
35 return beams[0][1] # Return best sequenceBLEU for translation, ROUGE for summarization.
1from collections import Counter
2
3def compute_bleu(reference, candidate, max_n=4):
4 """Compute BLEU score"""
5 def get_ngrams(tokens, n):
6 return [tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1)]
7
8 precisions = []
9 for n in range(1, max_n + 1):
10 ref_ngrams = Counter(get_ngrams(reference, n))
11 cand_ngrams = Counter(get_ngrams(candidate, n))
12
13 matches = sum((ref_ngrams & cand_ngrams).values())
14 total = sum(cand_ngrams.values())
15
16 precision = matches / total if total > 0 else 0
17 precisions.append(precision)
18
19 # Geometric mean
20 if 0 in precisions:
21 return 0
22
23 import math
24 avg_log = sum(math.log(p) for p in precisions) / len(precisions)
25
26 # Brevity penalty
27 bp = min(1, math.exp(1 - len(reference) / len(candidate))) if len(candidate) > 0 else 0
28
29 return bp * math.exp(avg_log)Build a basic seq2seq model for sequence transformation.
1import numpy as np
2
3class SimpleSeq2Seq:
4 """Simplified seq2seq for demonstration"""
5
6 def __init__(self, vocab_size, embed_dim=32, hidden_dim=64):
7 self.vocab_size = vocab_size
8 self.embed_dim = embed_dim
9 self.hidden_dim = hidden_dim
10
11 # Initialize weights
12 np.random.seed(42)
13 self.embed = np.random.randn(vocab_size, embed_dim) * 0.1
14 self.enc_W = np.random.randn(embed_dim, hidden_dim) * 0.1
15 self.dec_W = np.random.randn(embed_dim + hidden_dim, hidden_dim) * 0.1
16 self.out_W = np.random.randn(hidden_dim, vocab_size) * 0.1
17
18 def encode(self, src_ids):
19 """Simple encoder: average embeddings"""
20 embeddings = self.embed[src_ids] # (seq_len, embed_dim)
21 context = np.tanh(embeddings.mean(axis=0) @ self.enc_W)
22 return context
23
24 def decode_step(self, token_id, hidden):
25 """Single decoder step"""
26 embedding = self.embed[token_id]
27 combined = np.concatenate([embedding, hidden])
28 new_hidden = np.tanh(combined @ self.dec_W)
29 logits = new_hidden @ self.out_W
30 return logits, new_hidden
31
32 def decode(self, context, max_len=10, start_token=1, end_token=2):
33 """Greedy decoding"""
34 hidden = context
35 output = [start_token]
36 token = start_token
37
38 for _ in range(max_len):
39 logits, hidden = self.decode_step(token, hidden)
40 token = np.argmax(logits)
41 if token == end_token:
42 break
43 output.append(token)
44
45 return output
46
47 def translate(self, src_ids):
48 """Full translation pipeline"""
49 context = self.encode(src_ids)
50 output = self.decode(context)
51 return output
52
53# Simple vocabulary
54vocab = ['<pad>', '<start>', '<end>', 'hello', 'world', 'how', 'are', 'you',
55 'good', 'morning', 'hi', 'there', 'fine', 'thanks', 'bye']
56word2id = {w: i for i, w in enumerate(vocab)}
57id2word = {i: w for w, i in word2id.items()}
58
59# Initialize model
60model = SimpleSeq2Seq(vocab_size=len(vocab))
61
62print("SEQUENCE-TO-SEQUENCE MODEL")
63print("=" * 60)
64
65# Test sequences
66test_inputs = [
67 ['hello', 'world'],
68 ['how', 'are', 'you'],
69 ['good', 'morning'],
70]
71
72print("\nTranslation Examples (random model):")
73print("-" * 60)
74for words in test_inputs:
75 src_ids = np.array([word2id[w] for w in words])
76 output_ids = model.translate(src_ids)
77 output_words = [id2word[i] for i in output_ids if i not in [0, 1, 2]]
78
79 print(f" Input: {words}")
80 print(f" Output IDs: {output_ids}")
81 print(f" Output: {output_words}")
82 print()
83
84# Show attention concept
85print("ATTENTION VISUALIZATION CONCEPT")
86print("-" * 60)
87src_len = 5
88tgt_len = 4
89attn_weights = np.random.dirichlet(np.ones(src_len), size=tgt_len)
90
91print("Simulated attention weights (target x source):")
92src_words = ['The', 'cat', 'sat', 'on', 'mat']
93tgt_words = ['Le', 'chat', 'assis', 'tapis']
94
95print(f"{'':8}", end='')
96for w in src_words:
97 print(f"{w:>8}", end='')
98print()
99for i, w in enumerate(tgt_words):
100 print(f"{w:8}", end='')
101 for j in range(src_len):
102 weight = attn_weights[i, j]
103 bars = '█' * int(weight * 10)
104 print(f"{bars:>8}", end='')
105 print()This demonstrates the core concepts of seq2seq models including encoding, decoding, and attention mechanisms.
In our next lesson, "Transformers and BERT," you'll learn about the transformer architecture that has revolutionized NLP with self-attention and pre-training.
Your progress is saved in this browser, on this device only.