Learn to identify and classify named entities like people, organizations, locations, and dates in text using rule-based and machine learning approaches.
Named Entity Recognition (NER) is a fundamental NLP task that identifies and classifies named entities in text into predefined categories such as person names, organizations, locations, dates, monetary values, and more.
NER powers many applications: extracting contacts from emails, identifying companies in news articles, building knowledge graphs, and enabling question answering systems. Modern NER systems combine rule-based patterns with machine learning models trained on annotated data.
In this lesson, you'll learn both approaches and understand when to use each.
Standard entity types and the BIO tagging scheme for sequence labeling.
Common Entity Types:
BIO Tagging Scheme:
1# Example BIO tagging
2text = "Apple Inc. was founded by Steve Jobs in Cupertino"
3tokens = text.split()
4tags = ["B-ORG", "I-ORG", "O", "O", "O", "B-PER", "I-PER", "O", "B-LOC"]
5
6# Parsed entities
7# Apple Inc. -> ORG
8# Steve Jobs -> PER
9# Cupertino -> LOCPattern matching for entity extraction using regular expressions and gazetteers.
1import re
2from typing import List, Tuple
3
4class RuleBasedNER:
5 def __init__(self):
6 # Patterns for different entity types
7 self.patterns = {
8 'DATE': [
9 r'\b\d{1,2}/\d{1,2}/\d{2,4}\b',
10 r'\b\d{1,2}-\d{1,2}-\d{2,4}\b',
11 r'\b(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+\d{4}\b',
12 r'\b\d{4}\b'
13 ],
14 'MONEY': [
15 r'\$[\d,]+(?:\.\d{2})?',
16 r'\b\d+(?:,\d{3})*(?:\.\d{2})?\s*(?:dollars|USD|EUR)\b'
17 ],
18 'EMAIL': [
19 r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
20 ],
21 'PHONE': [
22 r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b',
23 r'\(\d{3}\)\s*\d{3}[-.]?\d{4}'
24 ]
25 }
26
27 # Gazetteers (known entity lists)
28 self.gazetteers = {
29 'ORG': {'Google', 'Apple', 'Microsoft', 'Amazon', 'Facebook', 'Meta'},
30 'LOC': {'New York', 'London', 'Tokyo', 'Paris', 'California'},
31 }
32
33 def extract(self, text: str) -> List[Tuple[str, str, int, int]]:
34 entities = []
35
36 # Pattern matching
37 for entity_type, patterns in self.patterns.items():
38 for pattern in patterns:
39 for match in re.finditer(pattern, text, re.IGNORECASE):
40 entities.append((match.group(), entity_type, match.start(), match.end()))
41
42 # Gazetteer matching
43 for entity_type, terms in self.gazetteers.items():
44 for term in terms:
45 for match in re.finditer(re.escape(term), text, re.IGNORECASE):
46 entities.append((match.group(), entity_type, match.start(), match.end()))
47
48 return sorted(entities, key=lambda x: x[2])Conditional Random Fields (CRF) for sequence labeling NER.
1# Feature extraction for CRF
2def extract_features(tokens, idx):
3 """Extract features for a token at position idx"""
4 token = tokens[idx]
5
6 features = {
7 'word': token.lower(),
8 'is_capitalized': token[0].isupper() if token else False,
9 'is_all_caps': token.isupper(),
10 'is_numeric': token.isdigit(),
11 'prefix_2': token[:2].lower() if len(token) > 1 else token,
12 'suffix_2': token[-2:].lower() if len(token) > 1 else token,
13 'length': len(token),
14 }
15
16 # Context features
17 if idx > 0:
18 features['prev_word'] = tokens[idx-1].lower()
19 features['prev_capitalized'] = tokens[idx-1][0].isupper()
20 else:
21 features['BOS'] = True # Beginning of sentence
22
23 if idx < len(tokens) - 1:
24 features['next_word'] = tokens[idx+1].lower()
25 features['next_capitalized'] = tokens[idx+1][0].isupper()
26 else:
27 features['EOS'] = True # End of sentence
28
29 return features
30
31# With sklearn-crfsuite
32from sklearn_crfsuite import CRF
33
34crf = CRF(
35 algorithm='lbfgs',
36 c1=0.1,
37 c2=0.1,
38 max_iterations=100
39)
40
41# X_train: List of List of feature dicts
42# y_train: List of List of labels
43crf.fit(X_train, y_train)
44predictions = crf.predict(X_test)spaCy provides production-ready NER models.
1import spacy
2
3# Load pre-trained model
4nlp = spacy.load("en_core_web_sm")
5
6text = "Apple CEO Tim Cook announced new products in San Francisco yesterday."
7doc = nlp(text)
8
9# Extract entities
10for ent in doc.ents:
11 print(f"{ent.text} -> {ent.label_} ({ent.start_char}:{ent.end_char})")
12
13# Output:
14# Apple -> ORG (0:5)
15# Tim Cook -> PERSON (10:18)
16# San Francisco -> GPE (51:64)
17# yesterday -> DATE (65:74)
18
19# Custom entity patterns
20from spacy.matcher import Matcher
21
22matcher = Matcher(nlp.vocab)
23pattern = [{"LOWER": "iphone"}, {"IS_DIGIT": True}]
24matcher.add("PRODUCT", [pattern])
25
26# Fine-tuning for custom entities
27# nlp.update(training_examples)Build a hybrid NER system combining rules and statistical methods.
1import re
2from collections import defaultdict
3
4class SimpleNER:
5 """Hybrid rule-based and statistical NER"""
6
7 def __init__(self):
8 # Regex patterns
9 self.patterns = {
10 'DATE': [
11 r'\b\d{1,2}/\d{1,2}/\d{2,4}\b',
12 r'\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2},?\s+\d{4}\b',
13 r'\b(?:yesterday|today|tomorrow)\b',
14 ],
15 'MONEY': [
16 r'\$[\d,]+(?:\.\d{2})?(?:\s*(?:million|billion|M|B))?',
17 r'\b\d+(?:\.\d+)?\s*(?:dollars|USD)\b',
18 ],
19 'PERCENT': [
20 r'\b\d+(?:\.\d+)?\s*%',
21 r'\b\d+(?:\.\d+)?\s*percent\b',
22 ],
23 }
24
25 # Known entities (gazetteers)
26 self.known_orgs = {
27 'apple', 'google', 'microsoft', 'amazon', 'facebook', 'meta',
28 'tesla', 'nvidia', 'ibm', 'intel', 'openai', 'anthropic'
29 }
30
31 self.known_locations = {
32 'new york', 'los angeles', 'san francisco', 'london', 'paris',
33 'tokyo', 'california', 'texas', 'usa', 'uk', 'china'
34 }
35
36 # Title words that often precede person names
37 self.title_words = {'mr', 'mrs', 'ms', 'dr', 'prof', 'ceo', 'cto', 'president'}
38
39 def extract_pattern_entities(self, text):
40 """Extract entities using regex patterns"""
41 entities = []
42 for ent_type, patterns in self.patterns.items():
43 for pattern in patterns:
44 for match in re.finditer(pattern, text, re.IGNORECASE):
45 entities.append({
46 'text': match.group(),
47 'type': ent_type,
48 'start': match.start(),
49 'end': match.end(),
50 'method': 'pattern'
51 })
52 return entities
53
54 def extract_gazetteer_entities(self, text):
55 """Extract entities using known entity lists"""
56 entities = []
57 text_lower = text.lower()
58
59 # Organizations
60 for org in self.known_orgs:
61 idx = text_lower.find(org)
62 if idx != -1:
63 # Get original case
64 original = text[idx:idx+len(org)]
65 entities.append({
66 'text': original,
67 'type': 'ORG',
68 'start': idx,
69 'end': idx + len(org),
70 'method': 'gazetteer'
71 })
72
73 # Locations
74 for loc in self.known_locations:
75 idx = text_lower.find(loc)
76 if idx != -1:
77 original = text[idx:idx+len(loc)]
78 entities.append({
79 'text': original,
80 'type': 'LOC',
81 'start': idx,
82 'end': idx + len(loc),
83 'method': 'gazetteer'
84 })
85
86 return entities
87
88 def extract_capitalized_sequences(self, text):
89 """Extract capitalized word sequences as potential entities"""
90 entities = []
91 # Match sequences of capitalized words
92 pattern = r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b'
93
94 for match in re.finditer(pattern, text):
95 seq = match.group()
96 words = seq.lower().split()
97
98 # Skip if first word is common
99 if words[0] in {'the', 'a', 'an', 'in', 'on', 'at', 'for'}:
100 continue
101
102 # Check for title words -> likely PERSON
103 if words[0] in self.title_words:
104 ent_type = 'PER'
105 # Two capitalized words often a person name
106 elif len(words) == 2 and all(w[0].isupper() for w in match.group().split()):
107 ent_type = 'PER'
108 else:
109 ent_type = 'MISC'
110
111 entities.append({
112 'text': seq,
113 'type': ent_type,
114 'start': match.start(),
115 'end': match.end(),
116 'method': 'capitalization'
117 })
118
119 return entities
120
121 def extract(self, text):
122 """Extract all entities and merge"""
123 all_entities = []
124 all_entities.extend(self.extract_pattern_entities(text))
125 all_entities.extend(self.extract_gazetteer_entities(text))
126 all_entities.extend(self.extract_capitalized_sequences(text))
127
128 # Remove duplicates and overlaps (keep longer spans)
129 entities = self._resolve_overlaps(all_entities)
130 return sorted(entities, key=lambda x: x['start'])
131
132 def _resolve_overlaps(self, entities):
133 """Remove overlapping entities, keeping longer spans"""
134 if not entities:
135 return []
136
137 # Sort by start position, then by length (descending)
138 sorted_ents = sorted(entities, key=lambda x: (x['start'], -(x['end'] - x['start'])))
139
140 result = []
141 last_end = -1
142
143 for ent in sorted_ents:
144 if ent['start'] >= last_end:
145 result.append(ent)
146 last_end = ent['end']
147
148 return result
149
150# Test the NER system
151ner = SimpleNER()
152
153test_texts = [
154 "Apple CEO Tim Cook announced a $2.5 billion investment in San Francisco yesterday.",
155 "Microsoft and Google are competing in the AI market. Stock prices rose 15% in January 2024.",
156 "Dr. Sarah Johnson from New York presented at the conference on 03/15/2024.",
157]
158
159print("NAMED ENTITY RECOGNITION SYSTEM")
160print("=" * 70)
161
162for text in test_texts:
163 print(f"\nText: {text}")
164 print("-" * 70)
165
166 entities = ner.extract(text)
167
168 if entities:
169 print("Entities found:")
170 for ent in entities:
171 print(f" [{ent['type']:6}] '{ent['text']}' (pos: {ent['start']}-{ent['end']}, method: {ent['method']})")
172 else:
173 print(" No entities found")
174
175# Entity statistics
176print("\n" + "=" * 70)
177print("ENTITY TYPE DISTRIBUTION:")
178all_entities = []
179for text in test_texts:
180 all_entities.extend(ner.extract(text))
181
182type_counts = defaultdict(int)
183for ent in all_entities:
184 type_counts[ent['type']] += 1
185
186for ent_type, count in sorted(type_counts.items(), key=lambda x: -x[1]):
187 print(f" {ent_type}: {count}")This demonstrates a hybrid NER approach combining regex patterns, gazetteers, and capitalization heuristics to extract named entities from text.
In our next lesson, "Sentiment Analysis," you'll learn to analyze opinions and emotions in text using lexicon-based and machine learning approaches.
Your progress is saved in this browser, on this device only.