Master text classification with Naive Bayes, SVM, and ensemble methods for spam detection, sentiment analysis, and topic classification.
Text classification is one of the most common NLP tasks, powering spam filters, sentiment analyzers, content moderators, and topic categorizers. The goal is to assign predefined labels to text documents based on their content.
Modern text classification combines feature extraction (TF-IDF, embeddings) with machine learning algorithms. While deep learning has achieved state-of-the-art results, traditional methods like Naive Bayes and SVM remain popular for their speed, interpretability, and effectiveness on smaller datasets.
In this lesson, you'll implement these fundamental classifiers and learn when to use each approach.
Naive Bayes is surprisingly effective for text despite its "naive" assumption that features are independent.
Multinomial Naive Bayes:
1import numpy as np
2from collections import Counter, defaultdict
3
4class MultinomialNB:
5 def __init__(self, alpha=1.0):
6 self.alpha = alpha # Laplace smoothing
7 self.class_priors = {}
8 self.word_probs = {}
9 self.classes = []
10 self.vocab = set()
11
12 def fit(self, X, y):
13 """X is list of tokenized documents, y is labels"""
14 self.classes = list(set(y))
15
16 # Count words per class
17 class_word_counts = defaultdict(Counter)
18 class_doc_counts = Counter(y)
19
20 for doc, label in zip(X, y):
21 class_word_counts[label].update(doc)
22 self.vocab.update(doc)
23
24 # Compute priors and likelihoods
25 n_docs = len(y)
26 vocab_size = len(self.vocab)
27
28 for c in self.classes:
29 self.class_priors[c] = class_doc_counts[c] / n_docs
30 total_words = sum(class_word_counts[c].values())
31 self.word_probs[c] = {}
32 for word in self.vocab:
33 count = class_word_counts[c][word]
34 # Laplace smoothing
35 self.word_probs[c][word] = (count + self.alpha) / (total_words + self.alpha * vocab_size)
36
37 return self
38
39 def predict(self, X):
40 return [self._predict_one(doc) for doc in X]
41
42 def _predict_one(self, doc):
43 scores = {}
44 for c in self.classes:
45 score = np.log(self.class_priors[c])
46 for word in doc:
47 if word in self.word_probs[c]:
48 score += np.log(self.word_probs[c][word])
49 scores[c] = score
50 return max(scores, key=scores.get)SVMs find the optimal hyperplane separating classes, maximizing the margin between them.
Linear SVM for Text:
1from sklearn.svm import LinearSVC
2from sklearn.pipeline import Pipeline
3from sklearn.feature_extraction.text import TfidfVectorizer
4
5# SVM Pipeline
6svm_pipeline = Pipeline([
7 ('tfidf', TfidfVectorizer(max_features=5000, ngram_range=(1, 2))),
8 ('classifier', LinearSVC(C=1.0, max_iter=1000))
9])
10
11# Train
12svm_pipeline.fit(X_train, y_train)
13
14# Predict
15predictions = svm_pipeline.predict(X_test)
16
17# Feature importance (for linear SVM)
18feature_names = svm_pipeline['tfidf'].get_feature_names_out()
19coefficients = svm_pipeline['classifier'].coef_[0]
20top_positive = [feature_names[i] for i in coefficients.argsort()[-10:]]
21top_negative = [feature_names[i] for i in coefficients.argsort()[:10]]Proper evaluation is crucial for text classification.
Key Metrics:
1from sklearn.metrics import classification_report, confusion_matrix
2
3def evaluate_classifier(y_true, y_pred, labels=None):
4 """Comprehensive evaluation"""
5 print("Classification Report:")
6 print(classification_report(y_true, y_pred, labels=labels))
7
8 print("\nConfusion Matrix:")
9 cm = confusion_matrix(y_true, y_pred, labels=labels)
10 print(cm)
11
12 # Per-class analysis
13 for i, label in enumerate(labels or sorted(set(y_true))):
14 tp = cm[i, i]
15 fp = cm[:, i].sum() - tp
16 fn = cm[i, :].sum() - tp
17 precision = tp / (tp + fp) if (tp + fp) > 0 else 0
18 recall = tp / (tp + fn) if (tp + fn) > 0 else 0
19 print(f"\n{label}: Precision={precision:.3f}, Recall={recall:.3f}")Real-world text datasets are often imbalanced (e.g., rare spam vs common ham).
Strategies:
1from sklearn.utils.class_weight import compute_class_weight
2from imblearn.over_sampling import SMOTE
3from imblearn.under_sampling import RandomUnderSampler
4
5# 1. Class weights
6class_weights = compute_class_weight('balanced', classes=np.unique(y), y=y)
7weight_dict = dict(zip(np.unique(y), class_weights))
8
9# Use in classifier
10svm = LinearSVC(class_weight='balanced')
11nb = MultinomialNB() # No built-in support, use sampling
12
13# 2. Oversampling minority class (SMOTE)
14smote = SMOTE(random_state=42)
15X_resampled, y_resampled = smote.fit_resample(X_tfidf, y)
16
17# 3. Undersampling majority class
18rus = RandomUnderSampler(random_state=42)
19X_resampled, y_resampled = rus.fit_resample(X_tfidf, y)
20
21# 4. Threshold adjustment
22# For probabilistic classifiers, adjust decision threshold
23probs = classifier.predict_proba(X_test)[:, 1]
24threshold = 0.3 # Lower threshold to catch more positives
25predictions = (probs >= threshold).astype(int)Build a production-ready sentiment classifier with multiple algorithms.
1import numpy as np
2from collections import Counter, defaultdict
3
4# Sample dataset
5reviews = [
6 ("This product is amazing and works perfectly", "positive"),
7 ("Terrible quality, broke after one day", "negative"),
8 ("Great value for the price, highly recommend", "positive"),
9 ("Worst purchase ever, complete waste of money", "negative"),
10 ("Decent product, nothing special but works", "neutral"),
11 ("Absolutely love it, exceeded expectations", "positive"),
12 ("Do not buy this, cheap and poorly made", "negative"),
13 ("Average quality, does the job", "neutral"),
14 ("Best thing I ever bought, five stars", "positive"),
15 ("Disappointed, not as described", "negative"),
16 ("Works as expected, satisfied", "neutral"),
17 ("Outstanding quality and fast shipping", "positive"),
18]
19
20# Split data
21train_data = reviews[:9]
22test_data = reviews[9:]
23
24X_train = [r[0].lower().split() for r in train_data]
25y_train = [r[1] for r in train_data]
26X_test = [r[0].lower().split() for r in test_data]
27y_test = [r[1] for r in test_data]
28
29# Naive Bayes Implementation
30class NaiveBayesClassifier:
31 def __init__(self, alpha=1.0):
32 self.alpha = alpha
33 self.class_priors = {}
34 self.word_probs = {}
35 self.vocab = set()
36
37 def fit(self, X, y):
38 classes = list(set(y))
39 class_word_counts = defaultdict(Counter)
40 class_counts = Counter(y)
41
42 for doc, label in zip(X, y):
43 class_word_counts[label].update(doc)
44 self.vocab.update(doc)
45
46 vocab_size = len(self.vocab)
47 n_docs = len(y)
48
49 for c in classes:
50 self.class_priors[c] = class_counts[c] / n_docs
51 total = sum(class_word_counts[c].values())
52 self.word_probs[c] = {}
53 for word in self.vocab:
54 count = class_word_counts[c][word]
55 self.word_probs[c][word] = (count + self.alpha) / (total + self.alpha * vocab_size)
56
57 return self
58
59 def predict(self, X):
60 predictions = []
61 for doc in X:
62 scores = {}
63 for c in self.class_priors:
64 score = np.log(self.class_priors[c])
65 for word in doc:
66 if word in self.word_probs[c]:
67 score += np.log(self.word_probs[c][word])
68 scores[c] = score
69 predictions.append(max(scores, key=scores.get))
70 return predictions
71
72 def get_top_features(self, n=5):
73 """Get most indicative words per class"""
74 features = {}
75 for c in self.class_priors:
76 sorted_words = sorted(self.word_probs[c].items(), key=lambda x: -x[1])
77 features[c] = [w for w, _ in sorted_words[:n]]
78 return features
79
80print("SENTIMENT CLASSIFICATION SYSTEM")
81print("=" * 60)
82
83# Train classifier
84nb = NaiveBayesClassifier(alpha=1.0)
85nb.fit(X_train, y_train)
86
87print(f"Training samples: {len(X_train)}")
88print(f"Vocabulary size: {len(nb.vocab)}")
89print(f"Classes: {list(nb.class_priors.keys())}")
90
91# Predictions
92predictions = nb.predict(X_test)
93
94print("\nPREDICTIONS:")
95print("-" * 60)
96for doc, pred, actual in zip(test_data, predictions, y_test):
97 status = "Correct" if pred == actual else "Wrong"
98 print(f"Text: '{doc[0][:40]}...'")
99 print(f" Predicted: {pred}, Actual: {actual} [{status}]")
100
101# Accuracy
102accuracy = sum(p == a for p, a in zip(predictions, y_test)) / len(y_test)
103print(f"\nAccuracy: {accuracy:.1%}")
104
105# Feature importance
106print("\nTOP FEATURES PER CLASS:")
107print("-" * 60)
108top_features = nb.get_top_features(n=5)
109for cls, words in top_features.items():
110 print(f" {cls}: {words}")This demonstrates a complete text classification pipeline with Naive Bayes, including training, prediction, evaluation, and feature analysis.
In our next lesson, "Named Entity Recognition," you'll learn to identify and classify named entities like people, organizations, and locations in text.
Your progress is saved in this browser, on this device only.