Discover hidden themes in document collections using Latent Dirichlet Allocation (LDA) and other unsupervised topic modeling techniques.
Topic modeling automatically discovers abstract "topics" in document collections. Unlike classification which assigns predefined labels, topic modeling finds hidden patterns without supervision - making it invaluable for exploring large text corpora.
The most popular algorithm is Latent Dirichlet Allocation (LDA), which models each document as a mixture of topics and each topic as a distribution over words. This probabilistic framework captures the intuition that documents often discuss multiple themes.
In this lesson, you'll implement topic modeling and learn to interpret, evaluate, and visualize discovered topics.
LDA assumes documents are generated by a probabilistic process:
Key Components:
1from sklearn.decomposition import LatentDirichletAllocation
2from sklearn.feature_extraction.text import CountVectorizer
3
4# Prepare data
5vectorizer = CountVectorizer(max_features=1000, stop_words='english')
6doc_term_matrix = vectorizer.fit_transform(documents)
7
8# Fit LDA
9lda = LatentDirichletAllocation(
10 n_components=5, # Number of topics
11 max_iter=20,
12 learning_method='online',
13 random_state=42
14)
15doc_topic_matrix = lda.fit_transform(doc_term_matrix)
16
17# Get top words per topic
18feature_names = vectorizer.get_feature_names_out()
19for topic_idx, topic in enumerate(lda.components_):
20 top_words = [feature_names[i] for i in topic.argsort()[-10:]]
21 print(f"Topic {topic_idx}: {', '.join(top_words)}")Selecting the right number of topics (k) is crucial.
Methods:
1from gensim.models import CoherenceModel
2
3def evaluate_topic_models(corpus, dictionary, texts, k_range):
4 coherence_scores = []
5
6 for k in k_range:
7 lda_model = LdaModel(
8 corpus=corpus,
9 id2word=dictionary,
10 num_topics=k,
11 random_state=42
12 )
13
14 coherence = CoherenceModel(
15 model=lda_model,
16 texts=texts,
17 dictionary=dictionary,
18 coherence='c_v'
19 )
20 coherence_scores.append(coherence.get_coherence())
21
22 return coherence_scores
23
24# Find optimal k
25k_range = range(2, 15)
26scores = evaluate_topic_models(corpus, dictionary, texts, k_range)
27optimal_k = k_range[scores.index(max(scores))]Making sense of discovered topics requires examining multiple signals.
1def interpret_topic(model, topic_idx, feature_names, n_top_words=15):
2 """Get comprehensive topic information"""
3 topic = model.components_[topic_idx]
4
5 # Top words by probability
6 top_indices = topic.argsort()[-n_top_words:][::-1]
7 top_words = [(feature_names[i], topic[i]) for i in top_indices]
8
9 # Most representative documents
10 doc_topics = model.transform(doc_term_matrix)
11 top_docs = doc_topics[:, topic_idx].argsort()[-5:][::-1]
12
13 return {
14 'top_words': top_words,
15 'representative_docs': top_docs
16 }
17
18# Label topics based on top words
19topic_labels = {
20 0: 'Technology',
21 1: 'Politics',
22 2: 'Sports',
23 3: 'Business',
24 4: 'Entertainment'
25}Understanding how topics distribute across documents.
1def analyze_document(doc_idx, doc_topic_matrix, topic_labels):
2 """Analyze topic distribution for a document"""
3 topic_probs = doc_topic_matrix[doc_idx]
4
5 # Sort by probability
6 sorted_topics = sorted(
7 enumerate(topic_probs),
8 key=lambda x: x[1],
9 reverse=True
10 )
11
12 print(f"Document {doc_idx} topic distribution:")
13 for topic_idx, prob in sorted_topics[:3]:
14 label = topic_labels.get(topic_idx, f'Topic {topic_idx}')
15 print(f" {label}: {prob:.2%}")
16
17# Find documents with mixed topics
18def find_multi_topic_docs(doc_topic_matrix, threshold=0.2):
19 """Find documents with multiple significant topics"""
20 multi_topic = []
21 for i, probs in enumerate(doc_topic_matrix):
22 significant = sum(1 for p in probs if p > threshold)
23 if significant >= 2:
24 multi_topic.append(i)
25 return multi_topicBuild a topic model from scratch and analyze document topics.
1import numpy as np
2from collections import Counter
3import re
4
5class SimpleLDA:
6 """Simplified LDA using Gibbs sampling"""
7
8 def __init__(self, n_topics=5, n_iter=100, alpha=0.1, beta=0.01):
9 self.n_topics = n_topics
10 self.n_iter = n_iter
11 self.alpha = alpha # Document-topic prior
12 self.beta = beta # Topic-word prior
13
14 def preprocess(self, documents):
15 """Tokenize and build vocabulary"""
16 stopwords = {'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be',
17 'been', 'being', 'have', 'has', 'had', 'do', 'does',
18 'did', 'will', 'would', 'could', 'should', 'may',
19 'might', 'must', 'shall', 'can', 'to', 'of', 'in',
20 'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into',
21 'through', 'during', 'before', 'after', 'above', 'below',
22 'and', 'but', 'or', 'nor', 'so', 'yet', 'both', 'either',
23 'neither', 'not', 'only', 'own', 'same', 'than', 'too',
24 'very', 'just', 'also', 'now', 'this', 'that', 'these',
25 'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they'}
26
27 tokenized = []
28 for doc in documents:
29 words = re.findall(r'\b[a-z]+\b', doc.lower())
30 words = [w for w in words if w not in stopwords and len(w) > 2]
31 tokenized.append(words)
32
33 # Build vocabulary
34 all_words = [w for doc in tokenized for w in doc]
35 word_counts = Counter(all_words)
36 self.vocab = [w for w, c in word_counts.most_common(500) if c >= 2]
37 self.word2id = {w: i for i, w in enumerate(self.vocab)}
38
39 # Convert to word IDs
40 docs_as_ids = []
41 for doc in tokenized:
42 ids = [self.word2id[w] for w in doc if w in self.word2id]
43 docs_as_ids.append(ids)
44
45 return docs_as_ids
46
47 def fit(self, documents):
48 """Fit LDA model"""
49 self.docs = self.preprocess(documents)
50 n_docs = len(self.docs)
51 n_words = len(self.vocab)
52
53 # Initialize counts
54 self.doc_topic_counts = np.zeros((n_docs, self.n_topics))
55 self.topic_word_counts = np.zeros((self.n_topics, n_words))
56 self.topic_counts = np.zeros(self.n_topics)
57
58 # Initialize topic assignments randomly
59 self.assignments = []
60 for d, doc in enumerate(self.docs):
61 doc_assignments = []
62 for word_id in doc:
63 topic = np.random.randint(self.n_topics)
64 doc_assignments.append(topic)
65 self.doc_topic_counts[d, topic] += 1
66 self.topic_word_counts[topic, word_id] += 1
67 self.topic_counts[topic] += 1
68 self.assignments.append(doc_assignments)
69
70 # Gibbs sampling
71 for iteration in range(self.n_iter):
72 for d, doc in enumerate(self.docs):
73 for i, word_id in enumerate(doc):
74 old_topic = self.assignments[d][i]
75
76 # Remove current assignment
77 self.doc_topic_counts[d, old_topic] -= 1
78 self.topic_word_counts[old_topic, word_id] -= 1
79 self.topic_counts[old_topic] -= 1
80
81 # Sample new topic
82 probs = self._sample_probs(d, word_id, n_words)
83 new_topic = np.random.choice(self.n_topics, p=probs)
84
85 # Add new assignment
86 self.assignments[d][i] = new_topic
87 self.doc_topic_counts[d, new_topic] += 1
88 self.topic_word_counts[new_topic, word_id] += 1
89 self.topic_counts[new_topic] += 1
90
91 return self
92
93 def _sample_probs(self, doc_idx, word_id, n_words):
94 """Calculate topic sampling probabilities"""
95 probs = np.zeros(self.n_topics)
96
97 for k in range(self.n_topics):
98 # P(topic|document)
99 doc_term = (self.doc_topic_counts[doc_idx, k] + self.alpha)
100 # P(word|topic)
101 word_term = (self.topic_word_counts[k, word_id] + self.beta) / \
102 (self.topic_counts[k] + self.beta * n_words)
103 probs[k] = doc_term * word_term
104
105 return probs / probs.sum()
106
107 def get_topics(self, n_top_words=10):
108 """Get top words for each topic"""
109 topics = []
110 for k in range(self.n_topics):
111 top_indices = self.topic_word_counts[k].argsort()[-n_top_words:][::-1]
112 top_words = [self.vocab[i] for i in top_indices]
113 topics.append(top_words)
114 return topics
115
116 def get_doc_topics(self, doc_idx):
117 """Get topic distribution for a document"""
118 counts = self.doc_topic_counts[doc_idx] + self.alpha
119 return counts / counts.sum()
120
121# Sample documents
122documents = [
123 "Machine learning algorithms can learn from data and make predictions",
124 "Deep neural networks have revolutionized computer vision and NLP",
125 "Python is popular for data science and machine learning applications",
126 "The stock market showed gains after positive economic news",
127 "Companies reported strong earnings in the quarterly financial reports",
128 "Interest rates affect investment decisions and market performance",
129 "The basketball team won the championship after a close game",
130 "Football players prepare for the upcoming season with training",
131 "Tennis tournaments attract millions of viewers worldwide",
132 "New smartphone features include better cameras and faster processors",
133 "Technology companies are investing heavily in artificial intelligence",
134 "Software developers use various programming languages and tools"
135]
136
137print("TOPIC MODELING WITH LDA")
138print("=" * 60)
139
140# Fit model
141lda = SimpleLDA(n_topics=3, n_iter=50)
142lda.fit(documents)
143
144# Show topics
145print("\nDiscovered Topics:")
146print("-" * 60)
147topics = lda.get_topics(n_top_words=8)
148for i, topic_words in enumerate(topics):
149 print(f"\nTopic {i}: {', '.join(topic_words)}")
150
151# Document-topic distributions
152print("\n" + "=" * 60)
153print("Document-Topic Distributions:")
154print("-" * 60)
155for i, doc in enumerate(documents[:5]):
156 probs = lda.get_doc_topics(i)
157 dominant = np.argmax(probs)
158 print(f"\nDoc {i}: '{doc[:40]}...'")
159 print(f" Topics: {['T{}: {:.0%}'.format(k, p) for k, p in enumerate(probs)]}")
160 print(f" Dominant: Topic {dominant}")This demonstrates a simplified LDA implementation using Gibbs sampling to discover topics in a document collection.
In our next lesson, "Sequence-to-Sequence Models," you'll learn about encoder-decoder architectures for machine translation and text summarization.
Your progress is saved in this browser, on this device only.