Master pattern matching and text manipulation with Python's powerful regex engine. Learn to validate, extract, and transform text data professionally.
Regular expressions (regex) are one of the most powerful tools for text processing and pattern matching. They provide a concise and flexible way to search, match, and manipulate text data. From validating user input to parsing log files, regex is essential for any Python developer working with text.
Think of regex as a sophisticated search and replace system that can understand patterns rather than just literal text. Companies like Google, Amazon, and Facebook use regex extensively for data processing, log analysis, and content filtering across millions of documents.
In this comprehensive lesson, you'll master regex from basic patterns to advanced text processing, building professional tools for data validation, extraction, and transformation.
Regular expressions use special characters and patterns to match text. Understanding these building blocks is essential for effective text processing.
Basic Pattern Matching:
1import re
2
3# Literal matching
4text = "Python is awesome for data science"
5pattern = r"Python"
6match = re.search(pattern, text)
7if match:
8 print(f"Found '{match.group()}' at position {match.start()}")
9
10# Case-insensitive matching
11pattern = r"PYTHON"
12match = re.search(pattern, text, re.IGNORECASE)
13print(f"Case-insensitive match: {match.group()}" if match else "No match")
14
15# Multiple matches
16pattern = r"a"
17matches = re.findall(pattern, text)
18print(f"Found {len(matches)} occurrences of 'a': {matches}")Essential Metacharacters:
1# . matches any character (except newline)
2pattern = r"P.thon"
3text = "Python, Pithon, P@thon"
4matches = re.findall(pattern, text)
5print(f"Dot matches: {matches}") # ['Python', 'Pithon', 'P@thon']
6
7# ^ matches start of string, $ matches end
8email_pattern = r"^[a-zA-Z0-9]+@[a-zA-Z0-9]+.[a-zA-Z]{2,}$"
9emails = ["[email protected]", "invalid-email", "[email protected]"]
10for email in emails:
11 if re.match(email_pattern, email):
12 print(f"Valid email: {email}")
13
14# * means 0 or more, + means 1 or more, ? means 0 or 1
15pattern = r"colou?r" # Matches both "color" and "colour"
16text = "I like the color red and the colour blue"
17matches = re.findall(pattern, text)
18print(f"Color variations: {matches}")
19
20# \d for digits, \w for word characters, \s for whitespace
21phone_pattern = r"\d{3}-\d{3}-\d{4}"
22text = "Call me at 555-123-4567 or 555-987-6543"
23phones = re.findall(phone_pattern, text)
24print(f"Phone numbers: {phones}")Character Classes and Ranges:
1# Character classes [abc] matches a, b, or c
2vowel_pattern = r"[aeiou]"
3text = "Regular expressions are powerful"
4vowels = re.findall(vowel_pattern, text)
5print(f"Vowels found: {vowels}")
6
7# Ranges [a-z] matches any lowercase letter
8pattern = r"[A-Z][a-z]+" # Capital letter followed by lowercase
9text = "John Smith works at Google Inc"
10proper_nouns = re.findall(pattern, text)
11print(f"Proper nouns: {proper_nouns}")
12
13# Negated character class [^abc] matches anything except a, b, or c
14non_digit_pattern = r"[^\d]+"
15text = "Price is $29.99 for item #12345"
16non_digits = re.findall(non_digit_pattern, text)
17print(f"Non-digit sequences: {non_digits}")Quantifiers for Precise Matching:
1# {n} exactly n times, {n,m} between n and m times
2zip_pattern = r"\d{5}" # Exactly 5 digits
3text = "ZIP codes: 12345, 67890, 555, 123456"
4zip_codes = re.findall(zip_pattern, text)
5print(f"Valid ZIP codes: {zip_codes}")
6
7# {n,} n or more times
8word_pattern = r"\w{3,}" # 3 or more word characters
9text = "I am a Python developer"
10long_words = re.findall(word_pattern, text)
11print(f"Words 3+ chars: {long_words}")Key Regex Methods:
re.search() - Finds first matchre.findall() - Finds all matchesre.match() - Matches at string startre.sub() - Search and replacere.split() - Split string by patternGroups allow you to extract specific parts of matches and create more sophisticated patterns for complex text processing tasks.
Capturing Groups:
1import re
2
3# Basic groups with parentheses
4email_pattern = r"([a-zA-Z0-9]+)@([a-zA-Z0-9]+)\.([a-zA-Z]{2,})"
5text = "Contact [email protected] or [email protected]"
6
7matches = re.finditer(email_pattern, text)
8for match in matches:
9 print(f"Full email: {match.group(0)}") # Full match
10 print(f"Username: {match.group(1)}") # First group
11 print(f"Domain: {match.group(2)}") # Second group
12 print(f"TLD: {match.group(3)}") # Third group
13 print(f"All groups: {match.groups()}") # All groups as tuple
14 print("---")
15
16# Named groups for clarity
17phone_pattern = r"(?P<area>\d{3})-(?P<prefix>\d{3})-(?P<number>\d{4})"
18text = "Call 555-123-4567 for support"
19
20match = re.search(phone_pattern, text)
21if match:
22 print(f"Area code: {match.group('area')}")
23 print(f"Prefix: {match.group('prefix')}")
24 print(f"Number: {match.group('number')}")
25 print(f"Formatted: ({match.group('area')}) {match.group('prefix')}-{match.group('number')}")Non-Capturing Groups and Alternatives:
1# Non-capturing groups (?:...) don't create capture groups
2color_pattern = r"(?:red|blue|green) car"
3text = "I saw a red car and a blue car yesterday"
4matches = re.findall(color_pattern, text)
5print(f"Colored cars: {matches}")
6
7# Alternatives with |
8file_pattern = r"\w+\.(jpg|png|gif|pdf|docx)"
9text = "Files: image.jpg, document.pdf, photo.png, invalid.txt"
10files = re.findall(file_pattern, text)
11print(f"Valid file extensions: {files}")
12
13# Complex URL parsing
14url_pattern = r"https?://(?:www\.)?([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?"
15text = "Visit https://www.python.org or http://github.com/user/repo"
16urls = re.findall(url_pattern, text)
17print(f"URLs found: {re.findall(r'https?://[^\s]+', text)}")Lookahead and Lookbehind Assertions:
1# Positive lookahead (?=...)
2password_pattern = r"^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[@$!%*?&])[A-Za-z\d@$!%*?&]{8,}$"
3passwords = ["weak", "Strong1!", "NoSpecial1", "short!", "Perfect123!"]
4
5for pwd in passwords:
6 if re.match(password_pattern, pwd):
7 print(f"ā
Strong password: {pwd}")
8 else:
9 print(f"ā Weak password: {pwd}")
10
11# Positive lookbehind (?<=...)
12price_pattern = r"(?<=\$)\d+\.\d{2}"
13text = "Items cost $19.99, $299.50, and ā¬45.00"
14dollar_prices = re.findall(price_pattern, text)
15print(f"Dollar prices: {dollar_prices}")
16
17# Negative lookahead (?!...)
18not_python_pattern = r"\b(?!Python)\w+ing\b"
19text = "Learning Python programming and teaching Java coding"
20non_python_words = re.findall(not_python_pattern, text)
21print(f"Non-Python words ending in 'ing': {non_python_words}")Greedy vs. Non-Greedy Matching:
1# Greedy matching (default)
2html = '<div>Content 1</div> and <div>Content 2</div>'
3greedy_pattern = r"<div>.*</div>"
4greedy_match = re.findall(greedy_pattern, html)
5print(f"Greedy match: {greedy_match}")
6
7# Non-greedy matching with ?
8non_greedy_pattern = r"<div>.*?</div>"
9non_greedy_matches = re.findall(non_greedy_pattern, html)
10print(f"Non-greedy matches: {non_greedy_matches}")
11
12# Practical example: extracting quoted strings
13text = 'He said "Hello there" and she replied "How are you?"'
14quotes_greedy = re.findall(r'".*"', text)
15quotes_non_greedy = re.findall(r'".*?"', text)
16print(f"Greedy quotes: {quotes_greedy}")
17print(f"Non-greedy quotes: {quotes_non_greedy}")Advanced Group Techniques:
1# Backreferences
2duplicate_pattern = r"\b(\w+)\s+\1\b" # \1 refers to first group
3text = "This this is a test test of duplicate duplicate words"
4duplicates = re.findall(duplicate_pattern, text, re.IGNORECASE)
5print(f"Duplicate words: {duplicates}")
6
7# Conditional groups
8# Match dates in MM/DD/YYYY or MM-DD-YYYY format
9date_pattern = r"(\d{2})[/-](\d{2})[/-](\d{4})"
10dates = ["12/25/2023", "01-15-2024", "invalid-date"]
11for date in dates:
12 match = re.match(date_pattern, date)
13 if match:
14 month, day, year = match.groups()
15 print(f"Date: {month}/{day}/{year}")Benefits of Advanced Patterns:
Real-world text processing involves cleaning messy data, standardizing formats, and extracting meaningful information from various sources.
Data Cleaning and Standardization:
1import re
2from collections import Counter
3
4class TextCleaner:
5 def __init__(self):
6 # Compile patterns for efficiency
7 self.phone_pattern = re.compile(r'[\(\)\-\s\.]', re.IGNORECASE)
8 self.email_pattern = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')
9 self.whitespace_pattern = re.compile(r'\s+')
10 self.special_chars_pattern = re.compile(r'[^a-zA-Z0-9\s]')
11
12 def clean_phone_number(self, phone):
13 """Standardize phone number format"""
14 # Remove all non-digits
15 digits = re.sub(r'\D', '', phone)
16
17 # Format as (XXX) XXX-XXXX
18 if len(digits) == 10:
19 return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}"
20 elif len(digits) == 11 and digits[0] == '1':
21 return f"({digits[1:4]}) {digits[4:7]}-{digits[7:]}"
22 else:
23 return "Invalid phone number"
24
25 def extract_emails(self, text):
26 """Extract all valid email addresses"""
27 return self.email_pattern.findall(text)
28
29 def clean_text(self, text):
30 """Clean and normalize text"""
31 # Convert to lowercase
32 text = text.lower()
33
34 # Replace multiple whitespace with single space
35 text = self.whitespace_pattern.sub(' ', text)
36
37 # Remove extra punctuation (keep basic punctuation)
38 text = re.sub(r'[^ws.,!?-]', '', text)
39
40 # Strip leading/trailing whitespace
41 return text.strip()
42
43 def extract_urls(self, text):
44 """Extract URLs from text"""
45 url_pattern = re.compile(
46 r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
47 )
48 return url_pattern.findall(text)
49
50# Example usage
51cleaner = TextCleaner()
52
53# Phone number standardization
54phone_numbers = [
55 "555-123-4567",
56 "(555) 987-6543",
57 "555.456.7890",
58 "1-800-555-0199",
59 "invalid-phone"
60]
61
62print("š Phone Number Standardization:")
63for phone in phone_numbers:
64 cleaned = cleaner.clean_phone_number(phone)
65 print(f"{phone:<20} -> {cleaned}")
66
67# Email extraction
68messy_text = """
69Contact us at [email protected] for help.
70You can also reach [email protected] or
71[email protected] for more information.
72Invalid emails: notanemail, @invalid.com, user@
73"""
74
75print("\nš§ Email Extraction:")
76emails = cleaner.extract_emails(messy_text)
77for email in emails:
78 print(f"Found email: {email}")Advanced Text Processing:
1class TextProcessor:
2 def __init__(self):
3 # Common text patterns
4 self.sentence_pattern = re.compile(r'[.!?]+')
5 self.word_pattern = re.compile(r'\b\w+\b')
6 self.number_pattern = re.compile(r'\b\d+(?:\.\d+)?\b')
7
8 def extract_sentences(self, text):
9 """Split text into sentences"""
10 sentences = self.sentence_pattern.split(text)
11 return [s.strip() for s in sentences if s.strip()]
12
13 def get_word_frequency(self, text):
14 """Count word frequency"""
15 words = self.word_pattern.findall(text.lower())
16 return Counter(words)
17
18 def extract_numbers(self, text):
19 """Extract all numbers from text"""
20 return [float(num) for num in self.number_pattern.findall(text)]
21
22 def find_capitalized_words(self, text):
23 """Find all capitalized words (potential proper nouns)"""
24 pattern = re.compile(r'\b[A-Z][a-z]+\b')
25 return pattern.findall(text)
26
27 def mask_sensitive_data(self, text):
28 """Mask sensitive information"""
29 # Mask credit card numbers
30 text = re.sub(r'\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b',
31 'XXXX-XXXX-XXXX-XXXX', text)
32
33 # Mask SSNs
34 text = re.sub(r'\b\d{3}-\d{2}-\d{4}\b', 'XXX-XX-XXXX', text)
35
36 # Mask email addresses
37 text = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',
38 '[EMAIL_MASKED]', text)
39
40 return text
41
42# Example processing
43processor = TextProcessor()
44
45sample_text = """
46John Smith works at Google Inc. He can be reached at [email protected].
47His credit card number is 4532-1234-5678-9012 and SSN is 123-45-6789.
48The project costs $1,250.50 and will take 3.5 months to complete.
49Contact him at (555) 123-4567 for more details.
50"""
51
52print("š Advanced Text Processing:")
53print(f"Sentences: {len(processor.extract_sentences(sample_text))}")
54print(f"Capitalized words: {processor.find_capitalized_words(sample_text)}")
55print(f"Numbers found: {processor.extract_numbers(sample_text)}")
56print("\nš Masked sensitive data:")
57print(processor.mask_sensitive_data(sample_text))Log File Processing:
1class LogProcessor:
2 def __init__(self):
3 # Common log patterns
4 self.apache_pattern = re.compile(
5 r'(\S+) \S+ \S+ \[([^\]]+)\] "(\S+) (\S+) \S+" (\d+) (\d+|-)'
6 )
7 self.error_pattern = re.compile(r'\b(ERROR|CRITICAL|FATAL)\b', re.IGNORECASE)
8 self.ip_pattern = re.compile(r'\b(?:\d{1,3}\.){3}\d{1,3}\b')
9
10 def parse_apache_log(self, log_line):
11 """Parse Apache access log entry"""
12 match = self.apache_pattern.match(log_line)
13 if match:
14 ip, timestamp, method, path, status, size = match.groups()
15 return {
16 'ip': ip,
17 'timestamp': timestamp,
18 'method': method,
19 'path': path,
20 'status': int(status),
21 'size': int(size) if size != '-' else 0
22 }
23 return None
24
25 def find_errors(self, log_content):
26 """Find all error entries in logs"""
27 return self.error_pattern.findall(log_content)
28
29 def extract_ips(self, log_content):
30 """Extract all IP addresses"""
31 return list(set(self.ip_pattern.findall(log_content)))
32
33# Example log processing
34log_processor = LogProcessor()
35
36sample_log = '''
37127.0.0.1 - - [25/Dec/2023:10:00:01 +0000] "GET /index.html HTTP/1.1" 200 1234
38192.168.1.1 - - [25/Dec/2023:10:00:02 +0000] "POST /login HTTP/1.1" 404 567
3910.0.0.1 - - [25/Dec/2023:10:00:03 +0000] "GET /dashboard HTTP/1.1" 500 0
40ERROR: Database connection failed
41CRITICAL: Security breach detected from IP 192.168.1.1
42'''
43
44print("š Log Analysis:")
45for line in sample_log.strip().split('\n'):
46 parsed = log_processor.parse_apache_log(line)
47 if parsed:
48 print(f"IP: {parsed['ip']}, Status: {parsed['status']}, Path: {parsed['path']}")
49
50print(f"\nError levels found: {log_processor.find_errors(sample_log)}")
51print(f"IP addresses: {log_processor.extract_ips(sample_log)}")Text Transformation and Formatting:
1def transform_text(text):
2 """Various text transformations"""
3
4 # Convert camelCase to snake_case
5 camel_to_snake = re.sub(r'(?<!^)(?=[A-Z])', '_', text).lower()
6
7 # Convert snake_case to camelCase
8 snake_to_camel = re.sub(r'_([a-z])', lambda m: m.group(1).upper(), text)
9
10 # Title case words
11 title_case = re.sub(r'\b\w+\b', lambda m: m.group().capitalize(), text)
12
13 return {
14 'original': text,
15 'snake_case': camel_to_snake,
16 'camel_case': snake_to_camel,
17 'title_case': title_case
18 }
19
20# Example transformations
21test_cases = ["getUserName", "user_profile_data", "this is a test"]
22print("š Text Transformations:")
23for case in test_cases:
24 result = transform_text(case)
25 print(f"Original: {result['original']}")
26 print(f"Snake: {result['snake_case']}")
27 print(f"Camel: {result['camel_case']}")
28 print(f"Title: {result['title_case']}")
29 print("---")Professional Text Processing Benefits:
Professional regex usage requires understanding performance optimization, maintainability, and practical applications in enterprise environments.
Data Validation Framework:
1import re
2from typing import Dict, List, Optional, Tuple
3
4class ValidationRules:
5 """Comprehensive validation using regex patterns"""
6
7 def __init__(self):
8 # Compile patterns for performance
9 self.patterns = {
10 'email': re.compile(r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'),
11 'phone_us': re.compile(r'^\+?1?[\s.-]?\(?([0-9]{3})\)?[\s.-]?([0-9]{3})[\s.-]?([0-9]{4})$'),
12 'password_strong': re.compile(r'^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[@$!%*?&])[A-Za-z\d@$!%*?&]{8,}$'),
13 'url': re.compile(r'^https?://(?:[-\w.])+(?:[:\d]+)?(?:/(?:[\w/_.])*(?:\?(?:[\w&=%.])+)?(?:#(?:[\w.])+)?)?$'),
14 'credit_card': re.compile(r'^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13})$'),
15 'ssn': re.compile(r'^\d{3}-\d{2}-\d{4}$'),
16 'zip_code': re.compile(r'^\d{5}(?:-\d{4})?$'),
17 'ip_address': re.compile(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'),
18 'date_iso': re.compile(r'^\d{4}-\d{2}-\d{2}$'),
19 'username': re.compile(r'^[a-zA-Z0-9_]{3,20}$')
20 }
21
22 def validate(self, field_type: str, value: str) -> Tuple[bool, str]:
23 """Validate a value against a specific pattern"""
24 if field_type not in self.patterns:
25 return False, f"Unknown validation type: {field_type}"
26
27 if self.patterns[field_type].match(value):
28 return True, "Valid"
29 else:
30 return False, f"Invalid {field_type} format"
31
32 def validate_form(self, data: Dict[str, str]) -> Dict[str, Tuple[bool, str]]:
33 """Validate multiple fields"""
34 results = {}
35
36 # Define field mappings
37 field_mappings = {
38 'email': 'email',
39 'phone': 'phone_us',
40 'password': 'password_strong',
41 'website': 'url',
42 'username': 'username',
43 'zip': 'zip_code'
44 }
45
46 for field, value in data.items():
47 if field in field_mappings:
48 results[field] = self.validate(field_mappings[field], value)
49 else:
50 results[field] = (True, "No validation rule")
51
52 return results
53
54# Example usage
55validator = ValidationRules()
56
57# Test individual validations
58test_data = {
59 'email': '[email protected]',
60 'phone': '555-123-4567',
61 'password': 'SecurePass123!',
62 'website': 'https://www.example.com',
63 'username': 'john_doe123',
64 'zip': '12345-6789'
65}
66
67print("ā
Form Validation Results:")
68results = validator.validate_form(test_data)
69for field, (is_valid, message) in results.items():
70 status = "ā
" if is_valid else "ā"
71 print(f"{status} {field}: {test_data[field]} - {message}")Text Mining and Analysis:
1class TextAnalyzer:
2 """Professional text analysis using regex"""
3
4 def __init__(self):
5 # Sentiment indicators
6 self.positive_words = re.compile(
7 r'\b(excellent|great|awesome|fantastic|perfect|love|amazing|wonderful)\b',
8 re.IGNORECASE
9 )
10 self.negative_words = re.compile(
11 r'\b(terrible|awful|hate|horrible|worst|disappointing|bad)\b',
12 re.IGNORECASE
13 )
14
15 # Technical terms
16 self.tech_terms = re.compile(
17 r'\b(python|javascript|react|django|api|database|algorithm|framework)\b',
18 re.IGNORECASE
19 )
20
21 # Mention patterns
22 self.mentions = re.compile(r'@([a-zA-Z0-9_]+)')
23 self.hashtags = re.compile(r'#([a-zA-Z0-9_]+)')
24
25 def analyze_sentiment(self, text: str) -> Dict[str, int]:
26 """Basic sentiment analysis"""
27 positive_matches = len(self.positive_words.findall(text))
28 negative_matches = len(self.negative_words.findall(text))
29
30 return {
31 'positive_score': positive_matches,
32 'negative_score': negative_matches,
33 'sentiment': 'positive' if positive_matches > negative_matches
34 else 'negative' if negative_matches > positive_matches
35 else 'neutral'
36 }
37
38 def extract_entities(self, text: str) -> Dict[str, List[str]]:
39 """Extract various entities from text"""
40 return {
41 'mentions': self.mentions.findall(text),
42 'hashtags': self.hashtags.findall(text),
43 'tech_terms': list(set(self.tech_terms.findall(text))),
44 'urls': re.findall(r'https?://[^\s]+', text)
45 }
46
47 def get_reading_stats(self, text: str) -> Dict[str, int]:
48 """Calculate reading statistics"""
49 sentences = len(re.findall(r'[.!?]+', text))
50 words = len(re.findall(r'\b\w+\b', text))
51 characters = len(re.sub(r'\s', '', text))
52 paragraphs = len(re.split(r'\n\s*\n', text.strip()))
53
54 avg_words_per_sentence = round(words / sentences, 1) if sentences > 0 else 0
55
56 return {
57 'sentences': sentences,
58 'words': words,
59 'characters': characters,
60 'paragraphs': paragraphs,
61 'avg_words_per_sentence': avg_words_per_sentence
62 }
63
64# Example text analysis
65analyzer = TextAnalyzer()
66
67sample_review = """
68This Python framework is absolutely fantastic! I love how easy it is to use.
69The documentation is excellent and the API design is wonderful.
70Check out the tutorial at https://example.com/tutorial
71Thanks @john_doe for the recommendation! #Python #WebDev #Amazing
72"""
73
74print("š Text Analysis Results:")
75sentiment = analyzer.analyze_sentiment(sample_review)
76print(f"Sentiment: {sentiment['sentiment']} (positive: {sentiment['positive_score']}, negative: {sentiment['negative_score']})")
77
78entities = analyzer.extract_entities(sample_review)
79for entity_type, items in entities.items():
80 if items:
81 print(f"{entity_type.title()}: {items}")
82
83stats = analyzer.get_reading_stats(sample_review)
84print(f"\nReading Stats:")
85for stat, value in stats.items():
86 print(f" {stat.replace('_', ' ').title()}: {value}")Performance Optimization:
1import re
2import time
3from functools import lru_cache
4
5class OptimizedProcessor:
6 """Performance-optimized regex processing"""
7
8 def __init__(self):
9 # Pre-compile frequently used patterns
10 self.compiled_patterns = {
11 'email': re.compile(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'),
12 'phone': re.compile(r'\d{3}[-.\s]?\d{3}[-.\s]?\d{4}'),
13 'url': re.compile(r'https?://[^\s]+'),
14 }
15
16 @lru_cache(maxsize=1000)
17 def cached_validation(self, pattern_name: str, text: str) -> bool:
18 """Cache validation results for repeated checks"""
19 if pattern_name in self.compiled_patterns:
20 return bool(self.compiled_patterns[pattern_name].search(text))
21 return False
22
23 def batch_process(self, texts: List[str], pattern_name: str) -> List[bool]:
24 """Process multiple texts efficiently"""
25 pattern = self.compiled_patterns.get(pattern_name)
26 if not pattern:
27 return [False] * len(texts)
28
29 return [bool(pattern.search(text)) for text in texts]
30
31 def benchmark_processing(self, texts: List[str], iterations: int = 1000):
32 """Benchmark different processing approaches"""
33
34 # Method 1: Compile pattern each time
35 start_time = time.time()
36 for _ in range(iterations):
37 for text in texts:
38 re.search(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}', text)
39 non_compiled_time = time.time() - start_time
40
41 # Method 2: Pre-compiled pattern
42 pattern = re.compile(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}')
43 start_time = time.time()
44 for _ in range(iterations):
45 for text in texts:
46 pattern.search(text)
47 compiled_time = time.time() - start_time
48
49 # Method 3: Cached validation
50 start_time = time.time()
51 for _ in range(iterations):
52 for text in texts:
53 self.cached_validation('email', text)
54 cached_time = time.time() - start_time
55
56 return {
57 'non_compiled': round(non_compiled_time, 4),
58 'compiled': round(compiled_time, 4),
59 'cached': round(cached_time, 4)
60 }
61
62# Performance demonstration
63processor = OptimizedProcessor()
64
65test_texts = [
66 "Contact [email protected]",
67 "Invalid email format",
68 "Another email: [email protected]",
69 "No email here",
70 "Multiple emails: [email protected], [email protected]"
71]
72
73# Benchmark results
74benchmark_results = processor.benchmark_processing(test_texts, 100)
75print("ā” Performance Benchmark (100 iterations):")
76for method, time_taken in benchmark_results.items():
77 print(f" {method}: {time_taken}s")
78
79# Show efficiency improvement
80improvement = round((benchmark_results['non_compiled'] / benchmark_results['compiled']) - 1, 2)
81print(f"\nš Pre-compiled patterns are {improvement * 100:.0f}% faster")Best Practices for Production:
1class ProductionRegexHandler:
2 """Production-ready regex handling with proper error management"""
3
4 def __init__(self):
5 self.patterns = {}
6 self.compile_patterns()
7
8 def compile_patterns(self):
9 """Safely compile all patterns with error handling"""
10 pattern_definitions = {
11 'email': r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
12 'phone': r'^\+?1?[-.\s]?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}$',
13 'url': r'^https?://(?:[-\w.])+(?:[:\d]+)?(?:/(?:[\w/_.])*(?:\?(?:[\w&=%.])+)?(?:#(?:[\w.])+)?)?$'
14 }
15
16 for name, pattern in pattern_definitions.items():
17 try:
18 self.patterns[name] = re.compile(pattern)
19 except re.error as e:
20 print(f"Error compiling pattern '{name}': {e}")
21 # Use a simple fallback pattern
22 self.patterns[name] = re.compile(r'.*')
23
24 def safe_match(self, pattern_name: str, text: str) -> Optional[bool]:
25 """Safely match with comprehensive error handling"""
26 if not isinstance(text, str):
27 return None
28
29 try:
30 pattern = self.patterns.get(pattern_name)
31 if pattern:
32 return bool(pattern.match(text))
33 except (re.error, TypeError, AttributeError) as e:
34 print(f"Error in pattern matching: {e}")
35 return None
36
37 return None
38
39 def validate_with_logging(self, data: Dict[str, str]) -> Dict[str, Dict]:
40 """Validate with detailed logging for debugging"""
41 results = {}
42
43 for field, value in data.items():
44 result = {
45 'value': value,
46 'is_valid': False,
47 'pattern_used': field,
48 'error': None
49 }
50
51 try:
52 match_result = self.safe_match(field, value)
53 if match_result is not None:
54 result['is_valid'] = match_result
55 else:
56 result['error'] = "Pattern matching failed"
57 except Exception as e:
58 result['error'] = str(e)
59
60 results[field] = result
61
62 return results
63
64# Production usage example
65handler = ProductionRegexHandler()
66
67production_data = {
68 'email': '[email protected]',
69 'phone': '+1-555-123-4567',
70 'url': 'https://www.example.com/path?param=value'
71}
72
73print("š Production Validation Results:")
74validation_results = handler.validate_with_logging(production_data)
75for field, result in validation_results.items():
76 status = "ā
" if result['is_valid'] else "ā"
77 print(f"{status} {field}: {result['value']} - Valid: {result['is_valid']}")
78 if result['error']:
79 print(f" Error: {result['error']}")Professional Regex Best Practices:
Create a comprehensive text processing system for analyzing customer feedback
1# Complete Customer Feedback Analysis System
2import re
3import json
4from datetime import datetime
5from collections import Counter, defaultdict
6from typing import Dict, List, Tuple, Optional
7
8class CustomerFeedbackProcessor:
9 """Advanced text processing system for customer feedback analysis"""
10
11 def __init__(self):
12 self.setup_patterns()
13 self.sentiment_scores = defaultdict(int)
14 self.processed_feedback = []
15
16 def setup_patterns(self):
17 """Initialize all regex patterns for processing"""
18
19 # Contact information patterns
20 self.email_pattern = re.compile(
21 r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
22 )
23 self.phone_pattern = re.compile(
24 r'\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b'
25 )
26
27 # Product and issue identification
28 self.product_pattern = re.compile(
29 r'\b(premium|basic|pro|enterprise|starter)\s+(plan|package|subscription|service)\b',
30 re.IGNORECASE
31 )
32 self.issue_pattern = re.compile(
33 r'\b(bug|error|problem|issue|broken|not working|failed|crash)\b',
34 re.IGNORECASE
35 )
36
37 # Sentiment analysis patterns
38 self.positive_pattern = re.compile(
39 r'\b(excellent|fantastic|amazing|great|awesome|perfect|love|wonderful|outstanding|brilliant)\b',
40 re.IGNORECASE
41 )
42 self.negative_pattern = re.compile(
43 r'\b(terrible|awful|horrible|worst|hate|disappointed|frustrated|angry|useless|broken)\b',
44 re.IGNORECASE
45 )
46
47 # Priority and urgency detection
48 self.urgent_pattern = re.compile(
49 r'\b(urgent|asap|immediately|critical|emergency|high priority)\b',
50 re.IGNORECASE
51 )
52
53 # Request patterns
54 self.refund_pattern = re.compile(
55 r'\b(refund|money back|cancel subscription|return)\b',
56 re.IGNORECASE
57 )
58 self.feature_request_pattern = re.compile(
59 r'\b(feature request|suggestion|would like|please add|can you add)\b',
60 re.IGNORECASE
61 )
62
63 def extract_contact_info(self, text: str) -> Dict[str, List[str]]:
64 """Extract contact information from feedback"""
65 return {
66 'emails': self.email_pattern.findall(text),
67 'phones': self.phone_pattern.findall(text)
68 }
69
70 def classify_feedback_type(self, text: str) -> List[str]:
71 """Classify the type of feedback"""
72 classifications = []
73
74 if self.issue_pattern.search(text):
75 classifications.append('bug_report')
76
77 if self.refund_pattern.search(text):
78 classifications.append('refund_request')
79
80 if self.feature_request_pattern.search(text):
81 classifications.append('feature_request')
82
83 if self.urgent_pattern.search(text):
84 classifications.append('urgent')
85
86 if not classifications:
87 classifications.append('general_feedback')
88
89 return classifications
90
91 def analyze_sentiment(self, text: str) -> Dict[str, any]:
92 """Analyze sentiment of feedback"""
93 positive_matches = self.positive_pattern.findall(text)
94 negative_matches = self.negative_pattern.findall(text)
95
96 positive_score = len(positive_matches)
97 negative_score = len(negative_matches)
98
99 # Determine overall sentiment
100 if positive_score > negative_score:
101 sentiment = 'positive'
102 elif negative_score > positive_score:
103 sentiment = 'negative'
104 else:
105 sentiment = 'neutral'
106
107 return {
108 'sentiment': sentiment,
109 'positive_score': positive_score,
110 'negative_score': negative_score,
111 'positive_words': positive_matches,
112 'negative_words': negative_matches,
113 'confidence': abs(positive_score - negative_score)
114 }
115
116 def extract_product_mentions(self, text: str) -> List[str]:
117 """Extract mentioned products or services"""
118 return self.product_pattern.findall(text)
119
120 def clean_and_standardize(self, text: str) -> str:
121 """Clean and standardize feedback text"""
122 # Remove extra whitespace
123 text = re.sub(r'\s+', ' ', text)
124
125 # Remove excessive punctuation
126 text = re.sub(r'[!]{2,}', '!', text)
127 text = re.sub(r'[?]{2,}', '?', text)
128
129 # Standardize common abbreviations
130 text = re.sub(r'\bu\b', 'you', text, flags=re.IGNORECASE)
131 text = re.sub(r'\bur\b', 'your', text, flags=re.IGNORECASE)
132 text = re.sub(r'\bthx\b', 'thanks', text, flags=re.IGNORECASE)
133
134 return text.strip()
135
136 def process_feedback(self, feedback_text: str, customer_id: str = None) -> Dict:
137 """Process a single piece of feedback comprehensively"""
138
139 # Clean the text
140 cleaned_text = self.clean_and_standardize(feedback_text)
141
142 # Extract various components
143 contact_info = self.extract_contact_info(cleaned_text)
144 classifications = self.classify_feedback_type(cleaned_text)
145 sentiment_analysis = self.analyze_sentiment(cleaned_text)
146 product_mentions = self.extract_product_mentions(cleaned_text)
147
148 # Create processing result
149 result = {
150 'customer_id': customer_id,
151 'original_text': feedback_text,
152 'cleaned_text': cleaned_text,
153 'timestamp': datetime.now().isoformat(),
154 'contact_info': contact_info,
155 'classifications': classifications,
156 'sentiment': sentiment_analysis,
157 'product_mentions': product_mentions,
158 'word_count': len(cleaned_text.split()),
159 'priority': 'high' if 'urgent' in classifications else 'normal'
160 }
161
162 # Store for analysis
163 self.processed_feedback.append(result)
164
165 return result
166
167 def generate_analytics_report(self) -> Dict:
168 """Generate comprehensive analytics from processed feedback"""
169 if not self.processed_feedback:
170 return {'error': 'No feedback processed yet'}
171
172 # Sentiment distribution
173 sentiments = [fb['sentiment']['sentiment'] for fb in self.processed_feedback]
174 sentiment_counts = Counter(sentiments)
175
176 # Classification distribution
177 all_classifications = []
178 for fb in self.processed_feedback:
179 all_classifications.extend(fb['classifications'])
180 classification_counts = Counter(all_classifications)
181
182 # Product mention analysis
183 all_products = []
184 for fb in self.processed_feedback:
185 all_products.extend(fb['product_mentions'])
186 product_counts = Counter(all_products)
187
188 # Priority distribution
189 priorities = [fb['priority'] for fb in self.processed_feedback]
190 priority_counts = Counter(priorities)
191
192 # Average scores
193 avg_positive_score = sum(fb['sentiment']['positive_score'] for fb in self.processed_feedback) / len(self.processed_feedback)
194 avg_negative_score = sum(fb['sentiment']['negative_score'] for fb in self.processed_feedback) / len(self.processed_feedback)
195
196 return {
197 'total_feedback': len(self.processed_feedback),
198 'sentiment_distribution': dict(sentiment_counts),
199 'classification_distribution': dict(classification_counts),
200 'product_mentions': dict(product_counts.most_common(5)),
201 'priority_distribution': dict(priority_counts),
202 'average_sentiment_scores': {
203 'positive': round(avg_positive_score, 2),
204 'negative': round(avg_negative_score, 2)
205 },
206 'top_issues': [item for item, count in classification_counts.most_common(3)],
207 'analysis_timestamp': datetime.now().isoformat()
208 }
209
210def demo_feedback_system():
211 """Demonstrate the complete feedback processing system"""
212
213 processor = CustomerFeedbackProcessor()
214
215 # Sample feedback data
216 sample_feedbacks = [
217 {
218 'text': "I love the premium plan! It's fantastic and works perfectly. Great job team! Contact me at [email protected] if you need testimonials.",
219 'customer_id': 'CUST_001'
220 },
221 {
222 'text': "This is terrible! The basic package is broken and not working at all. I want a refund immediately! Call me at 555-123-4567 ASAP!",
223 'customer_id': 'CUST_002'
224 },
225 {
226 'text': "Feature request: can you please add dark mode to the pro service? It would be amazing! Thanks!",
227 'customer_id': 'CUST_003'
228 },
229 {
230 'text': "The enterprise subscription has a critical bug in the dashboard. This is urgent - our business is affected!",
231 'customer_id': 'CUST_004'
232 },
233 {
234 'text': "Overall good experience with the starter plan. Could be improved but works fine for our needs.",
235 'customer_id': 'CUST_005'
236 }
237 ]
238
239 print("š Processing Customer Feedback...")
240 print("=" * 50)
241
242 # Process each feedback
243 for i, feedback in enumerate(sample_feedbacks, 1):
244 print(f"\nš Feedback #{i}:")
245 print(f"Customer: {feedback['customer_id']}")
246 print(f"Text: {feedback['text'][:80]}...")
247
248 result = processor.process_feedback(feedback['text'], feedback['customer_id'])
249
250 print(f"ā
Processed Results:")
251 print(f" Sentiment: {result['sentiment']['sentiment']} (confidence: {result['sentiment']['confidence']})")
252 print(f" Classifications: {', '.join(result['classifications'])}")
253 print(f" Priority: {result['priority']}")
254 print(f" Products mentioned: {result['product_mentions']}")
255 if result['contact_info']['emails']:
256 print(f" Contact emails: {result['contact_info']['emails']}")
257 if result['contact_info']['phones']:
258 print(f" Contact phones: {result['contact_info']['phones']}")
259
260 # Generate analytics report
261 print("\n" + "=" * 50)
262 print("š ANALYTICS REPORT")
263 print("=" * 50)
264
265 analytics = processor.generate_analytics_report()
266
267 print(f"š Total Feedback Processed: {analytics['total_feedback']}")
268 print(f"\nš Sentiment Distribution:")
269 for sentiment, count in analytics['sentiment_distribution'].items():
270 percentage = (count / analytics['total_feedback']) * 100
271 print(f" {sentiment.capitalize()}: {count} ({percentage:.1f}%)")
272
273 print(f"\nš·ļø Top Issue Types:")
274 for i, issue in enumerate(analytics['top_issues'], 1):
275 count = analytics['classification_distribution'][issue]
276 print(f" {i}. {issue.replace('_', ' ').title()}: {count}")
277
278 print(f"\nšÆ Priority Breakdown:")
279 for priority, count in analytics['priority_distribution'].items():
280 print(f" {priority.capitalize()}: {count}")
281
282 print(f"\nš¦ Most Mentioned Products:")
283 for product, count in analytics['product_mentions'].items():
284 print(f" {product}: {count} mentions")
285
286 print(f"\nš” Key Insights:")
287 positive_ratio = analytics['sentiment_distribution'].get('positive', 0) / analytics['total_feedback']
288 if positive_ratio > 0.6:
289 print(" ⢠Overall customer satisfaction is high")
290 elif positive_ratio < 0.3:
291 print(" ⢠Customer satisfaction needs immediate attention")
292
293 urgent_count = analytics['classification_distribution'].get('urgent', 0)
294 if urgent_count > 0:
295 print(f" ⢠{urgent_count} urgent issues require immediate response")
296
297 bug_count = analytics['classification_distribution'].get('bug_report', 0)
298 if bug_count > 0:
299 print(f" ⢠{bug_count} bug reports need technical investigation")
300
301 print("\nā
Feedback Analysis Complete!")
302 print("\nSystem Benefits:")
303 print("⢠Automated sentiment analysis and classification")
304 print("⢠Priority-based issue routing")
305 print("⢠Contact information extraction")
306 print("⢠Product-specific feedback tracking")
307 print("⢠Real-time analytics and insights")
308
309# Run the comprehensive demonstration
310if __name__ == "__main__":
311 demo_feedback_system()Your progress is saved in this browser, on this device only.