Natural Language Processing Basics

1import nltk
2from nltk.tokenize import word_tokenize, sent_tokenize
3import spacy
4
5# Download required NLTK data
6nltk.download('punkt')
7nltk.download('stopwords')
8
9text = "Natural Language Processing is amazing! It's transforming AI."
10
11# Word tokenization
12words = word_tokenize(text)
13print(f"Words: {words}")
14# Output: ['Natural', 'Language', 'Processing', 'is', 'amazing', '!', 'It', "'s", 'transforming', 'AI', '.']
15
16# Sentence tokenization
17sentences = sent_tokenize(text)
18print(f"Sentences: {sentences}")
19
20# SpaCy tokenization (more sophisticated)
21nlp = spacy.load('en_core_web_sm')
22doc = nlp(text)
23tokens = [token.text for token in doc]
24print(f"SpaCy tokens: {tokens}")

1from nltk.stem import PorterStemmer, WordNetLemmatizer
2from nltk.corpus import stopwords
3
4# Lowercasing
5text_lower = text.lower()
6
7# Remove punctuation
8import string
9text_no_punct = text.translate(str.maketrans('', '', string.punctuation))
10
11# Stemming (crude reduction to root form)
12stemmer = PorterStemmer()
13stems = [stemmer.stem(word) for word in words]
14print(f"Stems: {stems}")
15# 'running' -> 'run', 'better' -> 'better'
16
17# Lemmatization (context-aware reduction)
18lemmatizer = WordNetLemmatizer()
19lemmas = [lemmatizer.lemmatize(word) for word in words]
20print(f"Lemmas: {lemmas}")
21# 'running' -> 'run', 'better' -> 'good'
22
23# Remove stopwords
24stop_words = set(stopwords.words('english'))
25filtered_words = [w for w in words if w.lower() not in stop_words]
26print(f"Filtered: {filtered_words}")

1import re
2
3def preprocess_text(text, lowercase=True, remove_punct=True,
4                   remove_stopwords=True, lemmatize=True):
5    """Complete text preprocessing pipeline"""
6
7    # Lowercase
8    if lowercase:
9        text = text.lower()
10
11    # Remove URLs
12    text = re.sub(r'http\S+|www\S+|https\S+', '', text)
13
14    # Remove mentions and hashtags
15    text = re.sub(r'@\w+|#\w+', '', text)
16
17    # Remove punctuation
18    if remove_punct:
19        text = text.translate(str.maketrans('', '', string.punctuation))
20
21    # Tokenize
22    tokens = word_tokenize(text)
23
24    # Remove stopwords
25    if remove_stopwords:
26        stop_words = set(stopwords.words('english'))
27        tokens = [t for t in tokens if t not in stop_words]
28
29    # Lemmatize
30    if lemmatize:
31        lemmatizer = WordNetLemmatizer()
32        tokens = [lemmatizer.lemmatize(t) for t in tokens]
33
34    return tokens
35
36# Example
37text = "I'm loving Natural Language Processing! #NLP @AI_enthusiast"
38tokens = preprocess_text(text)
39print(f"Preprocessed: {tokens}")

1from gensim.models import Word2Vec
2import numpy as np
3
4# Training data: list of tokenized sentences
5sentences = [
6    ['natural', 'language', 'processing'],
7    ['deep', 'learning', 'neural', 'networks'],
8    ['machine', 'learning', 'artificial', 'intelligence'],
9    ['text', 'classification', 'sentiment', 'analysis']
10]
11
12# Train Word2Vec model
13model = Word2Vec(
14    sentences,
15    vector_size=100,      # Embedding dimension
16    window=5,             # Context window size
17    min_count=1,          # Minimum word frequency
18    sg=1,                 # Skip-gram (0 for CBOW)
19    epochs=100
20)
21
22# Get word vector
23vector = model.wv['learning']
24print(f"Vector shape: {vector.shape}")
25print(f"Vector (first 10 dims): {vector[:10]}")
26
27# Find similar words
28similar = model.wv.most_similar('learning', topn=5)
29print(f"Similar to 'learning': {similar}")
30
31# Word arithmetic
32result = model.wv.most_similar(
33    positive=['king', 'woman'],
34    negative=['man'],
35    topn=1
36)
37print(f"king - man + woman = {result}")  # Should be close to 'queen'
38
39# Compute similarity
40similarity = model.wv.similarity('deep', 'learning')
41print(f"Similarity(deep, learning): {similarity}")

1# Using pre-trained GloVe embeddings
2import torchtext
3from torchtext.vocab import GloVe
4
5# Load pre-trained GloVe vectors
6glove = GloVe(name='6B', dim=100)  # 6B tokens, 100-dim vectors
7
8# Get vector
9vector = glove['computer']
10print(f"GloVe vector shape: {vector.shape}")
11
12# Find nearest neighbors
13def get_nearest_neighbors(word, n=5):
14    word_vec = glove[word]
15    distances = [(w, torch.dist(word_vec, glove[w]))
16                 for w in glove.itos[:10000]]  # Check first 10k words
17    distances.sort(key=lambda x: x[1])
18    return distances[1:n+1]  # Exclude the word itself
19
20neighbors = get_nearest_neighbors('computer', n=5)
21print(f"Nearest to 'computer': {neighbors}")

1import torch
2import torch.nn as nn
3
4class WordEmbedding(nn.Module):
5    """Learnable word embedding layer"""
6    def __init__(self, vocab_size, embed_dim, padding_idx=0):
7        super().__init__()
8        self.embedding = nn.Embedding(
9            vocab_size, embed_dim, padding_idx=padding_idx
10        )
11
12    def forward(self, x):
13        # x: (batch_size, seq_len) - word indices
14        return self.embedding(x)  # (batch_size, seq_len, embed_dim)
15
16# Example
17vocab_size = 10000
18embed_dim = 300
19embedding = WordEmbedding(vocab_size, embed_dim)
20
21# Input: batch of sequences (word indices)
22input_ids = torch.randint(0, vocab_size, (32, 50))  # Batch=32, SeqLen=50
23embedded = embedding(input_ids)
24print(f"Embedded shape: {embedded.shape}")  # (32, 50, 300)

1from sklearn.feature_extraction.text import CountVectorizer
2
3documents = [
4    "I love machine learning",
5    "Deep learning is amazing",
6    "Machine learning and deep learning are related"
7]
8
9# Create BoW representation
10vectorizer = CountVectorizer()
11bow_matrix = vectorizer.fit_transform(documents)
12
13print(f"Vocabulary: {vectorizer.get_feature_names_out()}")
14print(f"BoW matrix shape: {bow_matrix.shape}")
15print(f"\nBoW representation (document 0):")
16print(bow_matrix[0].toarray())

1from sklearn.feature_extraction.text import TfidfVectorizer
2
3# TF-IDF representation
4tfidf_vectorizer = TfidfVectorizer()
5tfidf_matrix = tfidf_vectorizer.fit_transform(documents)
6
7print(f"TF-IDF matrix shape: {tfidf_matrix.shape}")
8print(f"\nTF-IDF representation (document 0):")
9print(tfidf_matrix[0].toarray())
10
11# Words with highest TF-IDF scores
12feature_names = tfidf_vectorizer.get_feature_names_out()
13for doc_idx in range(len(documents)):
14    scores = tfidf_matrix[doc_idx].toarray()[0]
15    top_indices = scores.argsort()[-3:][::-1]  # Top 3
16    top_words = [(feature_names[i], scores[i]) for i in top_indices]
17    print(f"\nDocument {doc_idx} top words: {top_words}")

1class TextRNN(nn.Module):
2    """RNN-based text classifier"""
3    def __init__(self, vocab_size, embed_dim, hidden_dim, num_classes,
4                 num_layers=1, dropout=0.5):
5        super().__init__()
6
7        self.embedding = nn.Embedding(vocab_size, embed_dim)
8        self.rnn = nn.RNN(
9            embed_dim, hidden_dim, num_layers,
10            batch_first=True, dropout=dropout if num_layers > 1 else 0
11        )
12        self.fc = nn.Linear(hidden_dim, num_classes)
13        self.dropout = nn.Dropout(dropout)
14
15    def forward(self, x):
16        # x: (batch, seq_len)
17        embedded = self.embedding(x)  # (batch, seq_len, embed_dim)
18
19        # RNN forward pass
20        output, hidden = self.rnn(embedded)  # output: (batch, seq_len, hidden_dim)
21
22        # Use last hidden state for classification
23        last_hidden = output[:, -1, :]  # (batch, hidden_dim)
24
25        # Classification
26        out = self.dropout(last_hidden)
27        logits = self.fc(out)  # (batch, num_classes)
28
29        return logits
30
31# Example
32model = TextRNN(vocab_size=10000, embed_dim=100, hidden_dim=256, num_classes=2)
33input_ids = torch.randint(0, 10000, (32, 50))
34logits = model(input_ids)
35print(f"Output logits shape: {logits.shape}")  # (32, 2)

1class SentimentLSTM(nn.Module):
2    """LSTM-based sentiment classifier"""
3    def __init__(self, vocab_size, embed_dim, hidden_dim, num_classes,
4                 num_layers=2, bidirectional=True, dropout=0.5):
5        super().__init__()
6
7        self.embedding = nn.Embedding(vocab_size, embed_dim)
8        self.lstm = nn.LSTM(
9            embed_dim, hidden_dim, num_layers,
10            batch_first=True, bidirectional=bidirectional,
11            dropout=dropout if num_layers > 1 else 0
12        )
13
14        # Account for bidirectional
15        lstm_output_dim = hidden_dim * 2 if bidirectional else hidden_dim
16
17        self.fc = nn.Sequential(
18            nn.Linear(lstm_output_dim, lstm_output_dim // 2),
19            nn.ReLU(),
20            nn.Dropout(dropout),
21            nn.Linear(lstm_output_dim // 2, num_classes)
22        )
23
24    def forward(self, x):
25        # Embed words
26        embedded = self.embedding(x)  # (batch, seq_len, embed_dim)
27
28        # LSTM
29        lstm_out, (hidden, cell) = self.lstm(embedded)
30
31        # Concatenate final forward and backward hidden states
32        if self.lstm.bidirectional:
33            hidden = torch.cat((hidden[-2], hidden[-1]), dim=1)
34        else:
35            hidden = hidden[-1]
36
37        # Classification
38        logits = self.fc(hidden)
39
40        return logits
41
42# Training loop
43def train_sentiment_classifier(model, train_loader, val_loader, num_epochs=10):
44    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
45    model = model.to(device)
46
47    criterion = nn.CrossEntropyLoss()
48    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
49
50    for epoch in range(num_epochs):
51        model.train()
52        train_loss = 0.0
53        correct = 0
54        total = 0
55
56        for batch in train_loader:
57            inputs, labels = batch['input_ids'].to(device), batch['labels'].to(device)
58
59            optimizer.zero_grad()
60            outputs = model(inputs)
61            loss = criterion(outputs, labels)
62            loss.backward()
63            optimizer.step()
64
65            train_loss += loss.item()
66            _, predicted = outputs.max(1)
67            total += labels.size(0)
68            correct += predicted.eq(labels).sum().item()
69
70        train_acc = 100. * correct / total
71
72        # Validation
73        model.eval()
74        val_loss = 0.0
75        correct = 0
76        total = 0
77
78        with torch.no_grad():
79            for batch in val_loader:
80                inputs, labels = batch['input_ids'].to(device), batch['labels'].to(device)
81                outputs = model(inputs)
82                loss = criterion(outputs, labels)
83
84                val_loss += loss.item()
85                _, predicted = outputs.max(1)
86                total += labels.size(0)
87                correct += predicted.eq(labels).sum().item()
88
89        val_acc = 100. * correct / total
90
91        print(f"Epoch {epoch+1}/{num_epochs}")
92        print(f"  Train Loss: {train_loss/len(train_loader):.4f}, Acc: {train_acc:.2f}%")
93        print(f"  Val Loss: {val_loss/len(val_loader):.4f}, Acc: {val_acc:.2f}%")

1from torch.utils.data import Dataset, DataLoader
2from torch.nn.utils.rnn import pad_sequence
3
4class TextDataset(Dataset):
5    """Dataset for text classification"""
6    def __init__(self, texts, labels, vocab, max_len=128):
7        self.texts = texts
8        self.labels = labels
9        self.vocab = vocab
10        self.max_len = max_len
11
12    def __len__(self):
13        return len(self.texts)
14
15    def __getitem__(self, idx):
16        # Tokenize and convert to indices
17        tokens = preprocess_text(self.texts[idx])
18        indices = [self.vocab.get(token, self.vocab['<UNK>'])
19                  for token in tokens[:self.max_len]]
20
21        return {
22            'input_ids': torch.tensor(indices),
23            'labels': torch.tensor(self.labels[idx])
24        }
25
26def collate_fn(batch):
27    """Collate function for batching with padding"""
28    input_ids = [item['input_ids'] for item in batch]
29    labels = torch.stack([item['labels'] for item in batch])
30
31    # Pad sequences
32    input_ids_padded = pad_sequence(input_ids, batch_first=True, padding_value=0)
33
34    return {
35        'input_ids': input_ids_padded,
36        'labels': labels
37    }
38
39# Build vocabulary
40def build_vocab(texts, min_freq=2):
41    """Build vocabulary from texts"""
42    from collections import Counter
43
44    word_counts = Counter()
45    for text in texts:
46        tokens = preprocess_text(text)
47        word_counts.update(tokens)
48
49    # Create vocab dict
50    vocab = {'<PAD>': 0, '<UNK>': 1}
51    for word, count in word_counts.items():
52        if count >= min_freq:
53            vocab[word] = len(vocab)
54
55    return vocab
56
57# Example usage
58texts = ["I love this movie!", "Terrible film, waste of time.", "Amazing plot and acting."]
59labels = [1, 0, 1]  # 1: positive, 0: negative
60
61vocab = build_vocab(texts)
62dataset = TextDataset(texts, labels, vocab)
63dataloader = DataLoader(dataset, batch_size=2, collate_fn=collate_fn)
64
65# Train model
66model = SentimentLSTM(
67    vocab_size=len(vocab),
68    embed_dim=100,
69    hidden_dim=256,
70    num_classes=2
71)
72
73# train_sentiment_classifier(model, train_loader, val_loader)

1# Using SpaCy for NER
2import spacy
3
4nlp = spacy.load('en_core_web_sm')
5
6text = "Apple Inc. is planning to open a new store in New York City next month."
7doc = nlp(text)
8
9print("Named Entities:")
10for ent in doc.ents:
11    print(f"  {ent.text}: {ent.label_}")
12
13# Custom NER with BiLSTM-CRF (architecture overview)
14class BiLSTM_CRF(nn.Module):
15    """BiLSTM-CRF for sequence labeling"""
16    def __init__(self, vocab_size, tagset_size, embed_dim, hidden_dim):
17        super().__init__()
18        self.embedding = nn.Embedding(vocab_size, embed_dim)
19        self.lstm = nn.LSTM(embed_dim, hidden_dim // 2, bidirectional=True, batch_first=True)
20        self.hidden2tag = nn.Linear(hidden_dim, tagset_size)
21        # CRF layer would go here
22
23    def forward(self, x):
24        embedded = self.embedding(x)
25        lstm_out, _ = self.lstm(embedded)
26        emissions = self.hidden2tag(lstm_out)
27        return emissions