Build NLP systems that work across multiple languages using cross-lingual embeddings, multilingual transformers, and machine translation.
As NLP applications expand globally, building systems that work across languages becomes essential. Multilingual NLP enables chatbots to serve users in multiple languages, search engines to retrieve documents regardless of language, and content moderation to work globally.
The key challenge is that different languages have different scripts, grammar structures, and vocabularies. Cross-lingual approaches learn representations that align languages in a shared embedding space, enabling zero-shot transfer from high-resource to low-resource languages.
In this lesson, you'll learn techniques for building language-agnostic NLP systems.
Align embeddings from different languages into a shared space.
Alignment Methods:
1def align_embeddings(source_embeds, target_embeds, translation_pairs):
2 """
3 Learn alignment matrix W to map source to target space.
4 source_embeds @ W ≈ target_embeds (for translation pairs)
5 """
6 # Get embedding matrices for translation pairs
7 src_words, tgt_words = zip(*translation_pairs)
8 X = np.array([source_embeds[w] for w in src_words])
9 Y = np.array([target_embeds[w] for w in tgt_words])
10
11 # Procrustes alignment: W = UV^T where X = UΣV^T (SVD of X^T Y)
12 U, S, Vt = np.linalg.svd(X.T @ Y)
13 W = U @ Vt
14
15 return W
16
17# After alignment
18def translate_word(word, source_embeds, W, target_embeds, target_vocab):
19 """Find nearest neighbor in target language"""
20 source_vec = source_embeds[word] @ W
21 similarities = target_embeds @ source_vec
22 nearest_idx = np.argmax(similarities)
23 return target_vocab[nearest_idx]This enables cross-lingual word retrieval and transfer learning.
mBERT is pre-trained on 104 languages with shared vocabulary.
1from transformers import AutoTokenizer, AutoModel
2
3class MultilingualEncoder:
4 """Encode text in any supported language"""
5
6 def __init__(self, model_name='bert-base-multilingual-cased'):
7 self.tokenizer = AutoTokenizer.from_pretrained(model_name)
8 self.model = AutoModel.from_pretrained(model_name)
9
10 def encode(self, texts, language=None):
11 """
12 Encode texts into embeddings.
13 Works for any of 104 supported languages.
14 """
15 inputs = self.tokenizer(
16 texts,
17 padding=True,
18 truncation=True,
19 return_tensors='pt'
20 )
21
22 with torch.no_grad():
23 outputs = self.model(**inputs)
24
25 # Use [CLS] token embedding
26 embeddings = outputs.last_hidden_state[:, 0, :]
27 return embeddings
28
29# Zero-shot cross-lingual transfer
30encoder = MultilingualEncoder()
31
32# Train classifier on English
33english_texts = ["This is great!", "Terrible experience"]
34english_labels = [1, 0]
35
36# Apply to French (zero-shot)
37french_texts = ["C'est génial!", "Expérience terrible"]
38# Same embeddings space allows transfermBERT learns language-agnostic representations automatically.
Identify the language of text for routing and processing.
1from collections import Counter
2
3class LanguageDetector:
4 """Simple n-gram based language detector"""
5
6 def __init__(self):
7 self.language_profiles = {}
8
9 def train(self, language, texts):
10 """Build character n-gram profile for language"""
11 ngrams = Counter()
12 for text in texts:
13 text = text.lower()
14 for i in range(len(text) - 2):
15 ngrams[text[i:i+3]] += 1
16
17 # Keep top n-grams as profile
18 total = sum(ngrams.values())
19 self.language_profiles[language] = {
20 ng: count / total for ng, count in ngrams.most_common(300)
21 }
22
23 def detect(self, text):
24 """Detect language of text"""
25 text = text.lower()
26 text_ngrams = Counter()
27 for i in range(len(text) - 2):
28 text_ngrams[text[i:i+3]] += 1
29
30 # Compare with each language profile
31 scores = {}
32 for lang, profile in self.language_profiles.items():
33 score = sum(
34 text_ngrams.get(ng, 0) * weight
35 for ng, weight in profile.items()
36 )
37 scores[lang] = score
38
39 return max(scores, key=scores.get)
40
41# Using transformers for better accuracy
42from transformers import pipeline
43classifier = pipeline('text-classification',
44 model='papluca/xlm-roberta-base-language-detection')
45result = classifier("Bonjour, comment allez-vous?")Strategies for languages with limited training data:
1# 1. Cross-lingual transfer from related languages
2def transfer_from_related(model, source_lang, target_lang):
3 """
4 Fine-tune on high-resource language,
5 apply to related low-resource language.
6 """
7 # Train on Hindi (high-resource)
8 # Apply to Nepali, Marathi (related, lower-resource)
9 pass
10
11# 2. Data augmentation
12def augment_low_resource(texts, labels):
13 """Augment limited training data"""
14 augmented = []
15
16 for text, label in zip(texts, labels):
17 augmented.append((text, label))
18
19 # Back-translation
20 translated = translate(text, to='en')
21 back = translate(translated, to=original_lang)
22 augmented.append((back, label))
23
24 # Synonym replacement using cross-lingual embeddings
25 synonyms = find_cross_lingual_synonyms(text)
26 for syn_text in synonyms:
27 augmented.append((syn_text, label))
28
29 return augmented
30
31# 3. Few-shot learning with multilingual models
32def few_shot_classify(examples, query, model):
33 """
34 Classify using few labeled examples.
35 Works across languages with mBERT.
36 """
37 # Encode examples and query
38 example_embs = model.encode([e[0] for e in examples])
39 query_emb = model.encode([query])
40
41 # Find nearest example
42 similarities = example_embs @ query_emb.T
43 nearest_idx = np.argmax(similarities)
44
45 return examples[nearest_idx][1] # Return labelCreate a classifier that works across multiple languages using cross-lingual techniques.
1import numpy as np
2from collections import Counter
3import re
4
5class MultilingualClassifier:
6 """Simple multilingual text classifier using character n-grams"""
7
8 def __init__(self):
9 self.class_profiles = {}
10 self.language_vocab = set()
11
12 def extract_features(self, text):
13 """Extract character n-gram features"""
14 text = text.lower()
15 # Character 2,3,4-grams
16 ngrams = []
17 for n in [2, 3, 4]:
18 for i in range(len(text) - n + 1):
19 ngrams.append(text[i:i+n])
20 return Counter(ngrams)
21
22 def train(self, texts, labels):
23 """Train on labeled texts from any language"""
24 # Aggregate features per class
25 class_features = {}
26
27 for text, label in zip(texts, labels):
28 if label not in class_features:
29 class_features[label] = Counter()
30
31 features = self.extract_features(text)
32 class_features[label].update(features)
33 self.language_vocab.update(features.keys())
34
35 # Normalize to probabilities
36 for label, features in class_features.items():
37 total = sum(features.values())
38 self.class_profiles[label] = {
39 f: (count + 1) / (total + len(self.language_vocab))
40 for f, count in features.items()
41 }
42
43 def predict(self, text):
44 """Predict class for text in any language"""
45 features = self.extract_features(text)
46
47 scores = {}
48 for label, profile in self.class_profiles.items():
49 score = 0
50 for ngram, count in features.items():
51 prob = profile.get(ngram, 1 / (len(self.language_vocab) + 1))
52 score += count * np.log(prob)
53 scores[label] = score
54
55 return max(scores, key=scores.get), scores
56
57class LanguageDetector:
58 """Detect language of text"""
59
60 def __init__(self):
61 self.lang_profiles = {}
62
63 def train(self, language, texts):
64 """Build profile for a language"""
65 ngrams = Counter()
66 for text in texts:
67 text = text.lower()
68 for n in [2, 3]:
69 for i in range(len(text) - n + 1):
70 ngrams[text[i:i+n]] += 1
71
72 total = sum(ngrams.values())
73 self.lang_profiles[language] = {
74 ng: count / total for ng, count in ngrams.most_common(200)
75 }
76
77 def detect(self, text):
78 """Detect language"""
79 text = text.lower()
80 text_ngrams = Counter()
81 for n in [2, 3]:
82 for i in range(len(text) - n + 1):
83 text_ngrams[text[i:i+n]] += 1
84
85 total = sum(text_ngrams.values())
86 text_profile = {ng: c / total for ng, c in text_ngrams.items()}
87
88 scores = {}
89 for lang, profile in self.lang_profiles.items():
90 # Cosine-like similarity
91 score = sum(
92 text_profile.get(ng, 0) * weight
93 for ng, weight in profile.items()
94 )
95 scores[lang] = score
96
97 return max(scores, key=scores.get), scores
98
99# Training data (sentiment in multiple languages)
100training_data = {
101 'english': [
102 ("This is amazing and wonderful!", "positive"),
103 ("I love this product so much!", "positive"),
104 ("Great experience, highly recommend!", "positive"),
105 ("This is terrible and awful!", "negative"),
106 ("Horrible service, very disappointed", "negative"),
107 ("Worst experience ever, never again", "negative"),
108 ],
109 'spanish': [
110 ("Esto es increíble y maravilloso!", "positive"),
111 ("Me encanta este producto!", "positive"),
112 ("Gran experiencia, muy recomendado!", "positive"),
113 ("Esto es terrible y horrible!", "negative"),
114 ("Servicio horrible, muy decepcionado", "negative"),
115 ("La peor experiencia, nunca más", "negative"),
116 ],
117 'french': [
118 ("C'est incroyable et merveilleux!", "positive"),
119 ("J'adore ce produit!", "positive"),
120 ("Excellente expérience, recommandé!", "positive"),
121 ("C'est terrible et affreux!", "negative"),
122 ("Service horrible, très déçu", "negative"),
123 ("Pire expérience jamais", "negative"),
124 ],
125}
126
127# Flatten for training
128all_texts = []
129all_labels = []
130lang_texts = {}
131
132for lang, data in training_data.items():
133 lang_texts[lang] = [t for t, _ in data]
134 for text, label in data:
135 all_texts.append(text)
136 all_labels.append(label)
137
138# Train classifier on ALL languages
139classifier = MultilingualClassifier()
140classifier.train(all_texts, all_labels)
141
142# Train language detector
143detector = LanguageDetector()
144for lang, texts in lang_texts.items():
145 detector.train(lang, texts)
146
147print("MULTILINGUAL NLP SYSTEM")
148print("=" * 60)
149
150# Test sentences (including languages not seen)
151test_sentences = [
152 "This movie is fantastic!",
153 "Este restaurante es muy malo",
154 "J'adore cette chanson!",
155 "Das ist wunderbar!", # German (not trained)
156 "Pessima qualità", # Italian (not trained)
157]
158
159print("\nCross-Lingual Sentiment Classification:")
160print("-" * 60)
161
162for sentence in test_sentences:
163 lang, _ = detector.detect(sentence)
164 pred, scores = classifier.predict(sentence)
165
166 print(f"\nText: '{sentence}'")
167 print(f" Detected Language: {lang}")
168 print(f" Sentiment: {pred}")
169 conf = np.exp(scores[pred]) / sum(np.exp(s) for s in scores.values())
170 print(f" Confidence: {conf:.1%}")
171
172print("\n" + "=" * 60)
173print("ZERO-SHOT TRANSFER DEMONSTRATION")
174print("-" * 60)
175
176# Train ONLY on English
177eng_classifier = MultilingualClassifier()
178eng_texts = [t for t, _ in training_data['english']]
179eng_labels = [l for _, l in training_data['english']]
180eng_classifier.train(eng_texts, eng_labels)
181
182# Test on other languages
183transfer_tests = [
184 ("Me encanta!", "Spanish"),
185 ("C'est terrible!", "French"),
186 ("Muy malo", "Spanish"),
187]
188
189print("\nTrained ONLY on English, testing on other languages:")
190for text, lang in transfer_tests:
191 pred, _ = eng_classifier.predict(text)
192 print(f" [{lang}] '{text}' -> {pred}")This demonstrates multilingual text classification using character n-grams, which naturally capture patterns across languages with similar scripts.
In our final lesson, "NLP Project Best Practices," you'll learn how to build production-ready NLP systems with proper evaluation, deployment, and monitoring.
Your progress is saved in this browser, on this device only.