Learn traditional text representation methods including bag of words, TF-IDF, and n-gram features for document classification and information retrieval.
Before deep learning dominated NLP, traditional text representation methods like Bag of Words (BoW) and TF-IDF powered search engines, spam filters, and document classifiers. These methods remain valuable today as strong baselines and for interpretable models.
The core idea is converting variable-length text into fixed-length numerical vectors that machine learning algorithms can process. BoW treats documents as unordered collections of words, while TF-IDF weighs words by their importance across the corpus.
In this lesson, you'll implement these fundamental techniques from scratch, understand their mathematical foundations, and learn when to use each approach in practice.
Bag of Words represents text as a vector of word counts, ignoring word order and grammar.
1from collections import Counter
2import numpy as np
3
4class BagOfWords:
5 def __init__(self):
6 self.vocabulary = {}
7
8 def fit(self, documents):
9 word_set = set()
10 for doc in documents:
11 word_set.update(doc.lower().split())
12 self.vocabulary = {w: i for i, w in enumerate(sorted(word_set))}
13 return self
14
15 def transform(self, documents):
16 vectors = []
17 for doc in documents:
18 vec = np.zeros(len(self.vocabulary))
19 for word in doc.lower().split():
20 if word in self.vocabulary:
21 vec[self.vocabulary[word]] += 1
22 vectors.append(vec)
23 return np.array(vectors)
24
25# Example
26docs = ["The cat sat on the mat", "The dog ran in the park"]
27bow = BagOfWords()
28bow.fit(docs)
29vectors = bow.transform(docs)
30print(f"Vocabulary: {bow.vocabulary}")
31print(f"Vectors shape: {vectors.shape}")TF-IDF (Term Frequency-Inverse Document Frequency) weighs words by their importance.
Formula:
1import numpy as np
2from collections import Counter
3
4class TfidfVectorizer:
5 def __init__(self):
6 self.vocabulary = {}
7 self.idf = {}
8
9 def fit(self, documents):
10 word_set = set()
11 doc_freq = Counter()
12
13 for doc in documents:
14 words = set(doc.lower().split())
15 word_set.update(words)
16 doc_freq.update(words)
17
18 self.vocabulary = {w: i for i, w in enumerate(sorted(word_set))}
19 N = len(documents)
20 self.idf = {w: np.log(N / (df + 1)) + 1 for w, df in doc_freq.items()}
21 return self
22
23 def transform(self, documents):
24 vectors = []
25 for doc in documents:
26 words = doc.lower().split()
27 tf = Counter(words)
28 vec = np.zeros(len(self.vocabulary))
29 for word, count in tf.items():
30 if word in self.vocabulary:
31 idx = self.vocabulary[word]
32 vec[idx] = (count / len(words)) * self.idf.get(word, 1)
33 vectors.append(vec)
34 return np.array(vectors)N-grams capture word sequences to preserve some word order information.
1def extract_ngrams(text, n=2):
2 words = text.lower().split()
3 ngrams = []
4 for i in range(len(words) - n + 1):
5 ngrams.append(' '.join(words[i:i+n]))
6 return ngrams
7
8text = "the quick brown fox jumps"
9print(f"Unigrams: {text.split()}")
10print(f"Bigrams: {extract_ngrams(text, 2)}")
11print(f"Trigrams: {extract_ngrams(text, 3)}")Using scikit-learn:
1from sklearn.feature_extraction.text import TfidfVectorizer
2
3vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=1000)
4X = vectorizer.fit_transform(documents)Combining TF-IDF with classifiers for text categorization:
1from sklearn.feature_extraction.text import TfidfVectorizer
2from sklearn.naive_bayes import MultinomialNB
3from sklearn.pipeline import Pipeline
4
5# Create pipeline
6pipeline = Pipeline([
7 ('tfidf', TfidfVectorizer(max_features=5000, ngram_range=(1,2))),
8 ('classifier', MultinomialNB())
9])
10
11# Train and predict
12pipeline.fit(X_train, y_train)
13predictions = pipeline.predict(X_test)Build a document classifier using TF-IDF features.
1import numpy as np
2from collections import Counter
3
4# Sample documents
5documents = [
6 "machine learning is amazing for data science",
7 "deep learning neural networks are powerful",
8 "python is great for machine learning",
9 "cooking recipes are delicious and healthy",
10 "baking bread requires patience and skill",
11 "neural networks learn from data patterns"
12]
13labels = ['tech', 'tech', 'tech', 'food', 'food', 'tech']
14
15# TF-IDF Implementation
16class SimpleTfidf:
17 def __init__(self, max_features=100):
18 self.max_features = max_features
19 self.vocabulary = {}
20 self.idf = {}
21
22 def fit_transform(self, docs):
23 # Build vocabulary
24 word_freq = Counter()
25 doc_freq = Counter()
26 for doc in docs:
27 words = doc.lower().split()
28 word_freq.update(words)
29 doc_freq.update(set(words))
30
31 # Top words by frequency
32 top_words = [w for w, _ in word_freq.most_common(self.max_features)]
33 self.vocabulary = {w: i for i, w in enumerate(top_words)}
34
35 # IDF
36 N = len(docs)
37 self.idf = {w: np.log(N / (doc_freq[w] + 1)) + 1 for w in self.vocabulary}
38
39 return self._transform(docs)
40
41 def _transform(self, docs):
42 vectors = []
43 for doc in docs:
44 words = doc.lower().split()
45 tf = Counter(words)
46 vec = np.zeros(len(self.vocabulary))
47 for word in self.vocabulary:
48 if word in tf:
49 idx = self.vocabulary[word]
50 vec[idx] = (tf[word] / len(words)) * self.idf[word]
51 vectors.append(vec)
52 return np.array(vectors)
53
54# Vectorize
55tfidf = SimpleTfidf(max_features=20)
56X = tfidf.fit_transform(documents)
57
58print("TF-IDF DOCUMENT CLASSIFICATION")
59print("=" * 60)
60print(f"Documents: {len(documents)}")
61print(f"Vocabulary size: {len(tfidf.vocabulary)}")
62print(f"Feature matrix shape: {X.shape}")
63print(f"\nVocabulary: {list(tfidf.vocabulary.keys())}")
64
65# Simple classifier (Naive Bayes-like)
66class SimpleClassifier:
67 def fit(self, X, y):
68 self.classes = list(set(y))
69 self.class_means = {}
70 for c in self.classes:
71 indices = [i for i, label in enumerate(y) if label == c]
72 self.class_means[c] = X[indices].mean(axis=0)
73
74 def predict(self, X):
75 predictions = []
76 for x in X:
77 best_class = min(self.classes,
78 key=lambda c: np.linalg.norm(x - self.class_means[c]))
79 predictions.append(best_class)
80 return predictions
81
82# Train and evaluate
83clf = SimpleClassifier()
84clf.fit(X, labels)
85predictions = clf.predict(X)
86
87print(f"\nPredictions: {predictions}")
88print(f"Actual: {labels}")
89print(f"Accuracy: {sum(p == l for p, l in zip(predictions, labels)) / len(labels):.2%}")This demonstrates TF-IDF vectorization and document classification using term frequency weighting.
In our next lesson, "Text Classification Techniques," you'll learn advanced classification methods combining these representations with neural networks.
Your progress is saved in this browser, on this device only.