Advanced NLP with Transformers

1from transformers import BertModel, BertTokenizer
2import torch
3
4# Load pre-trained BERT
5model_name = 'bert-base-uncased'
6tokenizer = BertTokenizer.from_pretrained(model_name)
7model = BertModel.from_pretrained(model_name)
8
9# Tokenize and encode text
10text = "Natural language processing with BERT is powerful!"
11inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True)
12
13print(f"Input IDs shape: {inputs['input_ids'].shape}")
14print(f"Attention mask shape: {inputs['attention_mask'].shape}")
15
16# Forward pass
17with torch.no_grad():
18    outputs = model(**inputs)
19
20# Extract representations
21last_hidden_state = outputs.last_hidden_state  # (batch, seq_len, hidden_dim)
22pooled_output = outputs.pooler_output  # (batch, hidden_dim) - [CLS] token
23
24print(f"\nLast hidden state shape: {last_hidden_state.shape}")
25print(f"Pooled output shape: {pooled_output.shape}")

1from transformers import BertForMaskedLM
2
3# Load BERT for MLM
4mlm_model = BertForMaskedLM.from_pretrained('bert-base-uncased')
5
6# Prepare masked input
7text = "Paris is the [MASK] of France."
8inputs = tokenizer(text, return_tensors='pt')
9
10# Predict masked token
11with torch.no_grad():
12    outputs = mlm_model(**inputs)
13    predictions = outputs.logits
14
15# Get predicted token
16mask_token_index = torch.where(inputs['input_ids'] == tokenizer.mask_token_id)[1]
17predicted_token_id = predictions[0, mask_token_index].argmax(axis=-1)
18predicted_token = tokenizer.decode(predicted_token_id)
19
20print(f"Masked sentence: {text}")
21print(f"Predicted word: {predicted_token}")  # Should be "capital"

1from transformers import BertForNextSentencePrediction
2
3nsp_model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased')
4
5# Test sentence pairs
6prompt = "The cat sat on the mat."
7next_sentence_correct = "It was sleeping peacefully."
8next_sentence_wrong = "The stock market crashed today."
9
10# Correct pair
11inputs = tokenizer(prompt, next_sentence_correct, return_tensors='pt')
12outputs = nsp_model(**inputs)
13logits = outputs.logits
14print(f"\nCorrect pair logits: {logits}")
15print(f"Prediction: {'IsNext' if logits[0][0] < logits[0][1] else 'NotNext'}")
16
17# Wrong pair
18inputs = tokenizer(prompt, next_sentence_wrong, return_tensors='pt')
19outputs = nsp_model(**inputs)
20logits = outputs.logits
21print(f"\nWrong pair logits: {logits}")
22print(f"Prediction: {'IsNext' if logits[0][0] < logits[0][1] else 'NotNext'}")

1from transformers import BertForSequenceClassification, AdamW, get_linear_schedule_with_warmup
2from torch.utils.data import DataLoader, Dataset
3import torch.nn.functional as F
4
5class TextClassificationDataset(Dataset):
6    """Dataset for text classification"""
7    def __init__(self, texts, labels, tokenizer, max_len=128):
8        self.texts = texts
9        self.labels = labels
10        self.tokenizer = tokenizer
11        self.max_len = max_len
12
13    def __len__(self):
14        return len(self.texts)
15
16    def __getitem__(self, idx):
17        text = self.texts[idx]
18        label = self.labels[idx]
19
20        encoding = self.tokenizer(
21            text,
22            max_length=self.max_len,
23            padding='max_length',
24            truncation=True,
25            return_tensors='pt'
26        )
27
28        return {
29            'input_ids': encoding['input_ids'].flatten(),
30            'attention_mask': encoding['attention_mask'].flatten(),
31            'labels': torch.tensor(label, dtype=torch.long)
32        }
33
34
35def train_bert_classifier(model, train_loader, val_loader, epochs=3, lr=2e-5):
36    """Fine-tune BERT for classification"""
37    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
38    model = model.to(device)
39
40    # Optimizer and scheduler
41    optimizer = AdamW(model.parameters(), lr=lr)
42    total_steps = len(train_loader) * epochs
43    scheduler = get_linear_schedule_with_warmup(
44        optimizer,
45        num_warmup_steps=total_steps // 10,
46        num_training_steps=total_steps
47    )
48
49    for epoch in range(epochs):
50        # Training
51        model.train()
52        train_loss = 0.0
53
54        for batch in train_loader:
55            input_ids = batch['input_ids'].to(device)
56            attention_mask = batch['attention_mask'].to(device)
57            labels = batch['labels'].to(device)
58
59            # Forward pass
60            outputs = model(
61                input_ids=input_ids,
62                attention_mask=attention_mask,
63                labels=labels
64            )
65
66            loss = outputs.loss
67
68            # Backward pass
69            optimizer.zero_grad()
70            loss.backward()
71            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
72            optimizer.step()
73            scheduler.step()
74
75            train_loss += loss.item()
76
77        avg_train_loss = train_loss / len(train_loader)
78
79        # Validation
80        model.eval()
81        val_loss = 0.0
82        correct = 0
83        total = 0
84
85        with torch.no_grad():
86            for batch in val_loader:
87                input_ids = batch['input_ids'].to(device)
88                attention_mask = batch['attention_mask'].to(device)
89                labels = batch['labels'].to(device)
90
91                outputs = model(
92                    input_ids=input_ids,
93                    attention_mask=attention_mask,
94                    labels=labels
95                )
96
97                val_loss += outputs.loss.item()
98                predictions = torch.argmax(outputs.logits, dim=-1)
99                correct += (predictions == labels).sum().item()
100                total += labels.size(0)
101
102        avg_val_loss = val_loss / len(val_loader)
103        accuracy = correct / total
104
105        print(f"Epoch {epoch + 1}/{epochs}")
106        print(f"  Train Loss: {avg_train_loss:.4f}")
107        print(f"  Val Loss: {avg_val_loss:.4f}")
108        print(f"  Val Accuracy: {accuracy:.4f}")
109
110
111# Example usage
112texts = ["I love this movie!", "Terrible film", "Amazing experience"]
113labels = [1, 0, 1]  # 1: positive, 0: negative
114
115tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
116model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)
117
118dataset = TextClassificationDataset(texts, labels, tokenizer)
119dataloader = DataLoader(dataset, batch_size=2, shuffle=True)
120
121# train_bert_classifier(model, train_loader, val_loader, epochs=3)

1from transformers import GPT2LMHeadModel, GPT2Tokenizer
2
3# Load GPT-2
4model_name = 'gpt2'
5tokenizer = GPT2Tokenizer.from_pretrained(model_name)
6model = GPT2LMHeadModel.from_pretrained(model_name)
7
8# Set padding token
9tokenizer.pad_token = tokenizer.eos_token
10
11def generate_text(prompt, max_length=50, num_return_sequences=3,
12                 temperature=0.7, top_k=50, top_p=0.95):
13    """Generate text with GPT-2"""
14
15    # Encode prompt
16    input_ids = tokenizer.encode(prompt, return_tensors='pt')
17
18    # Generate
19    output_sequences = model.generate(
20        input_ids,
21        max_length=max_length,
22        num_return_sequences=num_return_sequences,
23        temperature=temperature,  # Randomness (0.0-1.0)
24        top_k=top_k,  # Keep only top k tokens
25        top_p=top_p,  # Nucleus sampling
26        do_sample=True,
27        pad_token_id=tokenizer.eos_token_id
28    )
29
30    # Decode
31    generated_texts = []
32    for sequence in output_sequences:
33        text = tokenizer.decode(sequence, skip_special_tokens=True)
34        generated_texts.append(text)
35
36    return generated_texts
37
38
39# Generate text
40prompt = "Artificial intelligence is"
41generated = generate_text(prompt, max_length=30, num_return_sequences=3)
42
43print(f"Prompt: {prompt}")
44for i, text in enumerate(generated, 1):
45    print(f"\nGeneration {i}: {text}")

1def compare_sampling_strategies(prompt):
2    """Compare different sampling strategies"""
3
4    # Greedy (deterministic)
5    input_ids = tokenizer.encode(prompt, return_tensors='pt')
6    greedy_output = model.generate(input_ids, max_length=50, do_sample=False)
7    greedy_text = tokenizer.decode(greedy_output[0], skip_special_tokens=True)
8
9    # Beam search
10    beam_output = model.generate(
11        input_ids, max_length=50, num_beams=5,
12        early_stopping=True, do_sample=False
13    )
14    beam_text = tokenizer.decode(beam_output[0], skip_special_tokens=True)
15
16    # Top-k sampling
17    topk_output = model.generate(
18        input_ids, max_length=50, do_sample=True,
19        top_k=50, temperature=0.7
20    )
21    topk_text = tokenizer.decode(topk_output[0], skip_special_tokens=True)
22
23    # Nucleus (top-p) sampling
24    nucleus_output = model.generate(
25        input_ids, max_length=50, do_sample=True,
26        top_p=0.95, temperature=0.7
27    )
28    nucleus_text = tokenizer.decode(nucleus_output[0], skip_special_tokens=True)
29
30    return {
31        'greedy': greedy_text,
32        'beam': beam_text,
33        'top_k': topk_text,
34        'nucleus': nucleus_text
35    }
36
37# Compare
38results = compare_sampling_strategies("The future of AI is")
39for strategy, text in results.items():
40    print(f"\n{strategy.upper()}:\n{text}")

1from transformers import pipeline
2
3# 1. Sentiment Analysis
4sentiment_analyzer = pipeline("sentiment-analysis")
5result = sentiment_analyzer("I love using transformers!")
6print(f"Sentiment: {result}")
7
8# 2. Named Entity Recognition
9ner = pipeline("ner", aggregation_strategy="simple")
10text = "Apple Inc. was founded by Steve Jobs in Cupertino, California."
11entities = ner(text)
12print(f"\nNER: {entities}")
13
14# 3. Question Answering
15qa = pipeline("question-answering")
16context = "The Eiffel Tower is located in Paris, France. It was built in 1889."
17question = "Where is the Eiffel Tower?"
18answer = qa(question=question, context=context)
19print(f"\nQ&A: {answer}")
20
21# 4. Summarization
22summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
23long_text = """
24Artificial intelligence (AI) is intelligence demonstrated by machines,
25as opposed to natural intelligence displayed by humans or animals.
26Leading AI textbooks define the field as the study of "intelligent agents":
27any system that perceives its environment and takes actions that maximize
28its chance of achieving its goals.
29"""
30summary = summarizer(long_text, max_length=50, min_length=20)
31print(f"\nSummary: {summary}")
32
33# 5. Translation
34translator = pipeline("translation_en_to_fr", model="Helsinki-NLP/opus-mt-en-fr")
35translation = translator("Hello, how are you?")
36print(f"\nTranslation: {translation}")
37
38# 6. Text Generation
39generator = pipeline("text-generation", model="gpt2")
40generated = generator("Once upon a time", max_length=50, num_return_sequences=1)
41print(f"\nGenerated: {generated}")
42
43# 7. Zero-shot Classification
44classifier = pipeline("zero-shot-classification")
45text = "I love programming in Python!"
46candidate_labels = ["technology", "sports", "cooking", "music"]
47result = classifier(text, candidate_labels)
48print(f"\nZero-shot: {result}")

1from peft import LoraConfig, get_peft_model, TaskType
2
3# Configure LoRA
4lora_config = LoraConfig(
5    task_type=TaskType.SEQ_CLS,
6    r=8,  # Rank of update matrices
7    lora_alpha=32,  # Scaling factor
8    lora_dropout=0.1,
9    target_modules=["query", "value"]  # Which modules to adapt
10)
11
12# Apply LoRA to BERT
13base_model = BertForSequenceClassification.from_pretrained(
14    'bert-base-uncased', num_labels=2
15)
16model = get_peft_model(base_model, lora_config)
17
18# Print trainable parameters
19total_params = sum(p.numel() for p in model.parameters())
20trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
21
22print(f"Total parameters: {total_params:,}")
23print(f"Trainable parameters: {trainable_params:,}")
24print(f"Trainable %: {100 * trainable_params / total_params:.2f}%")
25# Only ~1-2% of parameters are trainable!

1import torch.nn as nn
2
3class AdapterLayer(nn.Module):
4    """Lightweight adapter for transfer learning"""
5    def __init__(self, hidden_size, adapter_size=64):
6        super().__init__()
7        self.down_project = nn.Linear(hidden_size, adapter_size)
8        self.activation = nn.ReLU()
9        self.up_project = nn.Linear(adapter_size, hidden_size)
10
11    def forward(self, hidden_states):
12        # Bottleneck architecture
13        down = self.activation(self.down_project(hidden_states))
14        up = self.up_project(down)
15        # Residual connection
16        return hidden_states + up
17
18
19# Add adapters to BERT layers
20class BertWithAdapters(nn.Module):
21    """BERT with adapter layers"""
22    def __init__(self, bert_model, adapter_size=64):
23        super().__init__()
24        self.bert = bert_model
25
26        # Add adapters after each transformer layer
27        self.adapters = nn.ModuleList([
28            AdapterLayer(bert_model.config.hidden_size, adapter_size)
29            for _ in range(bert_model.config.num_hidden_layers)
30        ])
31
32        # Freeze BERT parameters
33        for param in self.bert.parameters():
34            param.requires_grad = False
35
36    def forward(self, input_ids, attention_mask=None):
37        outputs = self.bert(input_ids, attention_mask=attention_mask,
38                           output_hidden_states=True)
39
40        hidden_states = outputs.hidden_states
41
42        # Apply adapters
43        adapted_states = []
44        for i, adapter in enumerate(self.adapters):
45            adapted = adapter(hidden_states[i + 1])
46            adapted_states.append(adapted)
47
48        return adapted_states[-1]  # Return final adapted layer

1def few_shot_classification(model, tokenizer, examples, query):
2    """Few-shot classification using prompting"""
3
4    # Build prompt with examples
5    prompt = "Classify the sentiment of the following texts:\n\n"
6
7    for text, label in examples:
8        prompt += f"Text: {text}\nSentiment: {label}\n\n"
9
10    prompt += f"Text: {query}\nSentiment:"
11
12    # Generate prediction
13    input_ids = tokenizer.encode(prompt, return_tensors='pt')
14    output = model.generate(
15        input_ids,
16        max_length=input_ids.shape[1] + 10,
17        num_return_sequences=1,
18        temperature=0.1  # Low temperature for more deterministic output
19    )
20
21    prediction = tokenizer.decode(output[0], skip_special_tokens=True)
22    return prediction.split("Sentiment:")[-1].strip()
23
24
25# Examples for few-shot learning
26examples = [
27    ("I love this product!", "Positive"),
28    ("Terrible experience", "Negative"),
29    ("It's okay, nothing special", "Neutral")
30]
31
32query = "This is the best thing ever!"
33
34# Load GPT model
35gpt_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
36gpt_model = GPT2LMHeadModel.from_pretrained('gpt2')
37gpt_tokenizer.pad_token = gpt_tokenizer.eos_token
38
39prediction = few_shot_classification(gpt_model, gpt_tokenizer, examples, query)
40print(f"Query: {query}")
41print(f"Predicted sentiment: {prediction}")

1# 1. Model Quantization
2from transformers import AutoModelForSequenceClassification
3import torch
4
5model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')
6
7# Dynamic quantization
8quantized_model = torch.quantization.quantize_dynamic(
9    model, {torch.nn.Linear}, dtype=torch.qint8
10)
11
12# Compare sizes
13original_size = sum(p.nelement() * p.element_size() for p in model.parameters())
14quantized_size = sum(p.nelement() * p.element_size() for p in quantized_model.parameters())
15
16print(f"Original size: {original_size / 1e6:.2f} MB")
17print(f"Quantized size: {quantized_size / 1e6:.2f} MB")
18print(f"Reduction: {(1 - quantized_size/original_size) * 100:.1f}%")
19
20
21# 2. ONNX Export
22from transformers import convert_graph_to_onnx
23
24convert_graph_to_onnx.convert(
25    framework="pt",
26    model=model,
27    tokenizer=tokenizer,
28    output="model.onnx",
29    opset=11
30)
31
32
33# 3. Inference Optimization with TorchScript
34traced_model = torch.jit.trace(
35    model,
36    (input_ids, attention_mask)
37)
38traced_model.save("model_traced.pt")