Word Embeddings: Word2Vec & GloVe

1import numpy as np
2
3# One-hot encoding for vocabulary of 5 words
4vocab = {'cat': 0, 'dog': 1, 'king': 2, 'queen': 3, 'apple': 4}
5vocab_size = len(vocab)
6
7def one_hot_encode(word, vocab, vocab_size):
8    """Create one-hot vector for word"""
9    vec = np.zeros(vocab_size)
10    vec[vocab[word]] = 1
11    return vec
12
13# Compare similarities
14cat_vec = one_hot_encode('cat', vocab, vocab_size)
15dog_vec = one_hot_encode('dog', vocab, vocab_size)
16king_vec = one_hot_encode('king', vocab, vocab_size)
17
18# Cosine similarity between vectors
19def cosine_similarity(a, b):
20    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-8)
21
22print("One-Hot Similarities:")
23print(f"cat vs dog: {cosine_similarity(cat_vec, dog_vec)}")  # 0.0
24print(f"cat vs king: {cosine_similarity(cat_vec, king_vec)}")  # 0.0
25# All pairs have 0 similarity - not useful!

1# Dense embeddings: words as low-dimensional vectors
2# Similar words have similar vectors
3
4embedding_dim = 4
5# Learned embeddings (hypothetical)
6embeddings = {
7    'cat':   np.array([0.8, 0.2, -0.1, 0.5]),   # Animal
8    'dog':   np.array([0.7, 0.3, -0.2, 0.4]),   # Animal (similar to cat)
9    'king':  np.array([-0.2, 0.9, 0.8, 0.1]),   # Royalty
10    'queen': np.array([-0.3, 0.8, 0.9, 0.2]),   # Royalty (similar to king)
11    'apple': np.array([0.1, -0.5, 0.2, 0.9]),   # Food
12}
13
14print("Dense Embedding Similarities:")
15print(f"cat vs dog: {cosine_similarity(embeddings['cat'], embeddings['dog']):.3f}")
16print(f"king vs queen: {cosine_similarity(embeddings['king'], embeddings['queen']):.3f}")
17print(f"cat vs king: {cosine_similarity(embeddings['cat'], embeddings['king']):.3f}")

1import numpy as np
2
3class Word2VecSkipGram:
4    """Simplified Word2Vec Skip-gram implementation"""
5
6    def __init__(self, vocab_size, embedding_dim):
7        self.vocab_size = vocab_size
8        self.embedding_dim = embedding_dim
9
10        # Initialize embeddings (input and output)
11        self.W_in = np.random.randn(vocab_size, embedding_dim) * 0.01
12        self.W_out = np.random.randn(embedding_dim, vocab_size) * 0.01
13
14    def forward(self, center_word_idx):
15        """
16        Forward pass: predict context word probabilities
17
18        Args:
19            center_word_idx: Index of center word
20
21        Returns:
22            Probability distribution over vocabulary
23        """
24        # Get embedding for center word
25        h = self.W_in[center_word_idx]  # (embedding_dim,)
26
27        # Compute scores for all words
28        scores = np.dot(h, self.W_out)  # (vocab_size,)
29
30        # Softmax to get probabilities
31        exp_scores = np.exp(scores - np.max(scores))
32        probs = exp_scores / np.sum(exp_scores)
33
34        return probs, h
35
36    def get_embedding(self, word_idx):
37        """Get word embedding"""
38        return self.W_in[word_idx]
39
40# Example usage
41vocab = {'the': 0, 'cat': 1, 'sat': 2, 'on': 3, 'mat': 4}
42model = Word2VecSkipGram(vocab_size=5, embedding_dim=10)
43
44# Get context probabilities for "cat"
45probs, embedding = model.forward(vocab['cat'])
46print(f"Embedding shape: {embedding.shape}")
47print(f"Context probabilities: {probs}")

1class Word2VecCBOW:
2    """Simplified CBOW implementation"""
3
4    def __init__(self, vocab_size, embedding_dim):
5        self.vocab_size = vocab_size
6        self.embedding_dim = embedding_dim
7
8        self.W_in = np.random.randn(vocab_size, embedding_dim) * 0.01
9        self.W_out = np.random.randn(embedding_dim, vocab_size) * 0.01
10
11    def forward(self, context_word_indices):
12        """
13        Forward pass: predict center word from context
14
15        Args:
16            context_word_indices: List of context word indices
17
18        Returns:
19            Probability distribution over vocabulary
20        """
21        # Average embeddings of context words
22        context_embeds = self.W_in[context_word_indices]  # (n_context, embed_dim)
23        h = np.mean(context_embeds, axis=0)  # (embedding_dim,)
24
25        # Compute scores and probabilities
26        scores = np.dot(h, self.W_out)
27        exp_scores = np.exp(scores - np.max(scores))
28        probs = exp_scores / np.sum(exp_scores)
29
30        return probs, h
31
32# Example: predict "cat" from context ["the", "sat"]
33cbow_model = Word2VecCBOW(vocab_size=5, embedding_dim=10)
34context = [vocab['the'], vocab['sat']]
35probs, _ = cbow_model.forward(context)
36print(f"Predicted word probabilities: {probs}")

1def negative_sampling_loss(pos_embedding, neg_embeddings, context_embedding):
2    """Compute negative sampling loss"""
3    # Positive sample: should have high similarity
4    pos_score = np.dot(pos_embedding, context_embedding)
5    pos_loss = -np.log(sigmoid(pos_score))
6
7    # Negative samples: should have low similarity
8    neg_scores = np.dot(neg_embeddings, context_embedding)
9    neg_loss = -np.sum(np.log(sigmoid(-neg_scores)))
10
11    return pos_loss + neg_loss
12
13def sigmoid(x):
14    return 1 / (1 + np.exp(-np.clip(x, -500, 500)))

1from gensim.models import Word2Vec
2
3# Training data: list of tokenized sentences
4sentences = [
5    ['the', 'cat', 'sat', 'on', 'the', 'mat'],
6    ['the', 'dog', 'ran', 'in', 'the', 'park'],
7    ['cats', 'and', 'dogs', 'are', 'pets']
8]
9
10# Train model
11model = Word2Vec(
12    sentences,
13    vector_size=100,    # Embedding dimension
14    window=5,           # Context window size
15    min_count=1,        # Minimum word frequency
16    sg=1,               # 1 for Skip-gram, 0 for CBOW
17    workers=4           # Parallel threads
18)
19
20# Get word vector
21cat_vector = model.wv['cat']
22
23# Find similar words
24similar = model.wv.most_similar('cat', topn=3)
25print(f"Words similar to 'cat': {similar}")

1from collections import defaultdict
2import numpy as np
3
4def build_cooccurrence_matrix(sentences, window_size=2):
5    """Build word co-occurrence matrix"""
6    # Build vocabulary
7    vocab = {}
8    idx = 0
9    for sentence in sentences:
10        for word in sentence:
11            if word not in vocab:
12                vocab[word] = idx
13                idx += 1
14
15    vocab_size = len(vocab)
16    cooccurrence = np.zeros((vocab_size, vocab_size))
17
18    # Count co-occurrences
19    for sentence in sentences:
20        for i, center in enumerate(sentence):
21            center_idx = vocab[center]
22
23            # Context window
24            start = max(0, i - window_size)
25            end = min(len(sentence), i + window_size + 1)
26
27            for j in range(start, end):
28                if i != j:
29                    context_idx = vocab[sentence[j]]
30                    # Weight by distance (closer = more weight)
31                    distance = abs(i - j)
32                    cooccurrence[center_idx, context_idx] += 1.0 / distance
33
34    return cooccurrence, vocab
35
36# Example
37sentences = [
38    ['the', 'cat', 'sat', 'on', 'mat'],
39    ['the', 'dog', 'ran', 'fast'],
40    ['cat', 'and', 'dog', 'played']
41]
42
43cooccur, vocab = build_cooccurrence_matrix(sentences)
44print(f"Vocabulary: {vocab}")
45print(f"Co-occurrence matrix shape: {cooccur.shape}")
46print(f"\nCo-occurrence counts for 'cat':")
47cat_idx = vocab['cat']
48for word, idx in vocab.items():
49    if cooccur[cat_idx, idx] > 0:
50        print(f"  {word}: {cooccur[cat_idx, idx]:.2f}")

1J = Σᵢⱼ f(Xᵢⱼ)(wᵢᵀw̃ⱼ + bᵢ + b̃ⱼ - log Xᵢⱼ)²

1def weighting_function(x, x_max=100, alpha=0.75):
2    """GloVe weighting function"""
3    if x < x_max:
4        return (x / x_max) ** alpha
5    return 1.0
6
7# Weighted loss penalizes frequent pairs less
8weights = np.vectorize(weighting_function)(cooccur)

1def load_glove_embeddings(filepath, embedding_dim=100):
2    """Load pre-trained GloVe embeddings"""
3    embeddings = {}
4
5    with open(filepath, 'r', encoding='utf-8') as f:
6        for line in f:
7            values = line.split()
8            word = values[0]
9            vector = np.array(values[1:], dtype='float32')
10            embeddings[word] = vector
11
12    return embeddings
13
14# Example: Using pre-trained GloVe
15# glove = load_glove_embeddings('glove.6B.100d.txt')
16# print(f"Loaded {len(glove)} word vectors")
17
18# Word similarity with GloVe
19def glove_similarity(word1, word2, embeddings):
20    """Compute cosine similarity between words"""
21    if word1 not in embeddings or word2 not in embeddings:
22        return None
23
24    v1 = embeddings[word1]
25    v2 = embeddings[word2]
26    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))

1def get_subwords(word, min_n=3, max_n=6):
2    """Extract character n-grams from word"""
3    # Add boundary markers
4    word = f"<{word}>"
5
6    subwords = []
7    word_len = len(word)
8
9    for n in range(min_n, min(max_n + 1, word_len + 1)):
10        for i in range(word_len - n + 1):
11            subwords.append(word[i:i+n])
12
13    return subwords
14
15# Example
16word = "running"
17subwords = get_subwords(word)
18print(f"Word: {word}")
19print(f"Subwords: {subwords}")
20# Output: ['<ru', 'run', 'unn', 'nni', 'nin', 'ing', 'ng>', ...]

1class SimpleFastText:
2    """Simplified FastText implementation"""
3
4    def __init__(self, vocab_size, embedding_dim, min_n=3, max_n=6):
5        self.embedding_dim = embedding_dim
6        self.min_n = min_n
7        self.max_n = max_n
8
9        # Subword embeddings (using hashing)
10        self.subword_embeddings = {}
11
12    def get_word_vector(self, word):
13        """Get word vector as sum of subword vectors"""
14        subwords = get_subwords(word, self.min_n, self.max_n)
15
16        # Sum subword embeddings
17        word_vector = np.zeros(self.embedding_dim)
18
19        for subword in subwords:
20            if subword not in self.subword_embeddings:
21                # Initialize randomly (in practice, learned)
22                self.subword_embeddings[subword] = np.random.randn(self.embedding_dim) * 0.1
23
24            word_vector += self.subword_embeddings[subword]
25
26        # Normalize
27        word_vector /= len(subwords)
28
29        return word_vector
30
31# Example: Handle OOV word
32model = SimpleFastText(vocab_size=10000, embedding_dim=100)
33
34# Known word
35known_vec = model.get_word_vector("running")
36
37# OOV word - still works!
38oov_vec = model.get_word_vector("sprinting")
39
40print(f"'running' vector shape: {known_vec.shape}")
41print(f"'sprinting' (OOV) vector shape: {oov_vec.shape}")

1# Morphological similarity example
2def morphological_similarity():
3    """FastText captures morphological relationships"""
4
5    # Words sharing subwords will be similar
6    examples = [
7        ("run", "running", "runner"),      # Same root
8        ("happy", "unhappy", "happiness"), # Prefix/suffix
9        ("compute", "computer", "computing")
10    ]
11
12    for group in examples:
13        print(f"Related words: {group}")
14        # FastText would find these similar due to shared subwords

1from gensim.models import FastText
2
3# Train FastText model
4sentences = [
5    ['the', 'quick', 'brown', 'fox'],
6    ['jumped', 'over', 'the', 'lazy', 'dog'],
7    ['foxes', 'are', 'quick', 'animals']
8]
9
10model = FastText(
11    sentences,
12    vector_size=100,
13    window=5,
14    min_count=1,
15    min_n=3,        # Minimum n-gram length
16    max_n=6,        # Maximum n-gram length
17    workers=4
18)
19
20# Works for OOV words!
21oov_vector = model.wv['quickfox']  # Not in training data
22print(f"OOV 'quickfox' vector shape: {oov_vector.shape}")

1import numpy as np
2
3class EmbeddingModel:
4    """Work with pre-trained word embeddings"""
5
6    def __init__(self, embeddings_dict):
7        self.embeddings = embeddings_dict
8        self.dim = len(next(iter(embeddings_dict.values())))
9
10    def get_vector(self, word, handle_oov='zero'):
11        """Get embedding for word"""
12        if word in self.embeddings:
13            return self.embeddings[word]
14
15        # Handle OOV
16        if handle_oov == 'zero':
17            return np.zeros(self.dim)
18        elif handle_oov == 'random':
19            return np.random.randn(self.dim) * 0.1
20        else:
21            return None
22
23    def similarity(self, word1, word2):
24        """Cosine similarity between words"""
25        v1 = self.get_vector(word1)
26        v2 = self.get_vector(word2)
27
28        if v1 is None or v2 is None:
29            return None
30
31        return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2) + 1e-8)
32
33    def most_similar(self, word, topn=5):
34        """Find most similar words"""
35        if word not in self.embeddings:
36            return []
37
38        target = self.embeddings[word]
39        similarities = []
40
41        for other_word, vector in self.embeddings.items():
42            if other_word != word:
43                sim = np.dot(target, vector) / (
44                    np.linalg.norm(target) * np.linalg.norm(vector) + 1e-8
45                )
46                similarities.append((other_word, sim))
47
48        return sorted(similarities, key=lambda x: -x[1])[:topn]
49
50    def analogy(self, a, b, c):
51        """Solve analogy: a is to b as c is to ?"""
52        # Vector arithmetic: result = b - a + c
53        if a not in self.embeddings or b not in self.embeddings or c not in self.embeddings:
54            return None
55
56        target = self.embeddings[b] - self.embeddings[a] + self.embeddings[c]
57
58        # Find closest word
59        best_word = None
60        best_sim = -1
61
62        for word, vector in self.embeddings.items():
63            if word not in [a, b, c]:
64                sim = np.dot(target, vector) / (
65                    np.linalg.norm(target) * np.linalg.norm(vector) + 1e-8
66                )
67                if sim > best_sim:
68                    best_sim = sim
69                    best_word = word
70
71        return best_word, best_sim

1def create_embedding_matrix(vocab, embeddings, embedding_dim):
2    """Create embedding matrix for vocabulary"""
3    vocab_size = len(vocab)
4    embedding_matrix = np.zeros((vocab_size, embedding_dim))
5
6    oov_count = 0
7    for word, idx in vocab.items():
8        if word in embeddings:
9            embedding_matrix[idx] = embeddings[word]
10        else:
11            # Random initialization for OOV
12            embedding_matrix[idx] = np.random.randn(embedding_dim) * 0.1
13            oov_count += 1
14
15    print(f"Vocabulary size: {vocab_size}")
16    print(f"OOV words: {oov_count} ({100*oov_count/vocab_size:.1f}%)")
17
18    return embedding_matrix
19
20# Example usage
21vocab = {'<PAD>': 0, '<UNK>': 1, 'the': 2, 'cat': 3, 'sat': 4}
22# embedding_matrix = create_embedding_matrix(vocab, glove_embeddings, 100)

1from sklearn.manifold import TSNE
2import matplotlib.pyplot as plt
3
4def visualize_embeddings(words, embeddings, perplexity=5):
5    """Visualize embeddings using t-SNE"""
6    # Get vectors for words
7    vectors = np.array([embeddings[w] for w in words if w in embeddings])
8    labels = [w for w in words if w in embeddings]
9
10    # Reduce to 2D
11    tsne = TSNE(n_components=2, perplexity=perplexity, random_state=42)
12    reduced = tsne.fit_transform(vectors)
13
14    # Plot
15    plt.figure(figsize=(12, 8))
16    plt.scatter(reduced[:, 0], reduced[:, 1], alpha=0.6)
17
18    for i, label in enumerate(labels):
19        plt.annotate(label, (reduced[i, 0], reduced[i, 1]))
20
21    plt.title('Word Embeddings Visualization (t-SNE)')
22    plt.xlabel('t-SNE dimension 1')
23    plt.ylabel('t-SNE dimension 2')
24    plt.grid(True, alpha=0.3)
25    plt.show()
26
27# visualize_embeddings(['king', 'queen', 'man', 'woman', 'prince', 'princess'], embeddings)