Implement automated performance testing and optimization workflows
Performance automation ensures applications meet SLA requirements through continuous testing, load simulation, and automated optimization. By implementing comprehensive performance testing workflows, teams can identify bottlenecks early, validate capacity planning, and maintain optimal system performance.
Why Performance Automation Matters:
Real-World Applications: Load testing APIs and web applications, performance benchmarking for releases, profiling code for optimization, stress testing infrastructure, capacity planning for scaling, and continuous performance monitoring.
Automated load testing simulates realistic user traffic to measure system performance, identify capacity limits, and validate scalability. Locust is a powerful Python-based load testing framework that enables distributed testing with code-based scenarios.
Locust Load Testing Framework:
1from locust import HttpUser, task, between
2from locust.env import Environment
3from locust.stats import stats_printer, stats_history
4import time
5from typing import Dict, List
6from dataclasses import dataclass
7import statistics
8
9@dataclass
10class PerformanceMetrics:
11 """Performance test metrics"""
12 endpoint: str
13 avg_response_time: float
14 max_response_time: float
15 min_response_time: float
16 percentile_95: float
17 percentile_99: float
18 requests_per_second: float
19 failure_rate: float
20 total_requests: int
21
22class APIUser(HttpUser):
23 """
24 Simulated user for load testing
25 """
26 wait_time = between(1, 3) # Wait 1-3 seconds between tasks
27
28 @task(3) # Weight 3 - runs 3x more often
29 def get_users(self):
30 """Test GET /api/users endpoint"""
31 self.client.get("/api/users", name="GET /api/users")
32
33 @task(2) # Weight 2
34 def get_user_by_id(self):
35 """Test GET /api/users/:id endpoint"""
36 user_id = random.randint(1, 1000)
37 self.client.get(
38 f"/api/users/{user_id}",
39 name="GET /api/users/:id"
40 )
41
42 @task(1) # Weight 1
43 def create_user(self):
44 """Test POST /api/users endpoint"""
45 payload = {
46 "name": f"User_{random.randint(1, 10000)}",
47 "email": f"user_{random.randint(1, 10000)}@example.com"
48 }
49 self.client.post(
50 "/api/users",
51 json=payload,
52 name="POST /api/users"
53 )
54
55 def on_start(self):
56 """Called when user starts - authenticate"""
57 response = self.client.post("/api/auth/login", json={
58 "username": "test_user",
59 "password": "test_password"
60 })
61 if response.status_code == 200:
62 self.token = response.json().get("token")
63 self.client.headers.update({"Authorization": f"Bearer {self.token}"})
64
65class LoadTestRunner:
66 """
67 Programmatic load test execution
68 """
69
70 def __init__(self, host: str):
71 self.host = host
72 self.env = Environment(user_classes=[APIUser])
73 self.env.host = host
74
75 def run_load_test(
76 self,
77 users: int = 100,
78 spawn_rate: int = 10,
79 duration: int = 60
80 ) -> Dict[str, PerformanceMetrics]:
81 """
82 Run load test programmatically
83
84 Args:
85 users: Total number of concurrent users
86 spawn_rate: Users spawned per second
87 duration: Test duration in seconds
88
89 Returns:
90 Performance metrics by endpoint
91 """
92 logger.info(f"š Starting load test: {users} users @ {spawn_rate}/s for {duration}s")
93
94 # Start load test
95 self.env.runner.start(user_count=users, spawn_rate=spawn_rate)
96
97 # Run for specified duration
98 time.sleep(duration)
99
100 # Stop test
101 self.env.runner.quit()
102
103 # Calculate metrics
104 metrics = {}
105 for name, stats in self.env.runner.stats.entries.items():
106 if isinstance(name, str):
107 metrics[name] = PerformanceMetrics(
108 endpoint=name,
109 avg_response_time=stats.avg_response_time,
110 max_response_time=stats.max_response_time,
111 min_response_time=stats.min_response_time,
112 percentile_95=stats.get_response_time_percentile(0.95),
113 percentile_99=stats.get_response_time_percentile(0.99),
114 requests_per_second=stats.total_rps,
115 failure_rate=stats.fail_ratio,
116 total_requests=stats.num_requests
117 )
118
119 return metrics
120
121# Example: Run load test
122runner = LoadTestRunner(host="https://api.example.com")
123metrics = runner.run_load_test(users=500, spawn_rate=50, duration=120)
124
125for endpoint, metric in metrics.items():
126 print(f"\nEndpoint: {endpoint}")
127 print(f" Avg Response: {metric.avg_response_time:.2f}ms")
128 print(f" 95th %ile: {metric.percentile_95:.2f}ms")
129 print(f" RPS: {metric.requests_per_second:.2f}")
130 print(f" Failures: {metric.failure_rate*100:.2f}%")Performance Metrics Analysis:
1import numpy as np
2from datetime import datetime, timedelta
3
4class PerformanceAnalyzer:
5 """Analyze performance test results"""
6
7 def __init__(self):
8 self.thresholds = {
9 'response_time_ms': 200,
10 'error_rate_percent': 1.0,
11 'throughput_rps': 100
12 }
13
14 def analyze_results(self, metrics: Dict[str, PerformanceMetrics]) -> Dict:
15 """
16 Analyze performance metrics against thresholds
17
18 Returns:
19 Analysis report with pass/fail status
20 """
21 analysis = {
22 'timestamp': datetime.now().isoformat(),
23 'overall_status': 'pass',
24 'endpoint_results': {},
25 'violations': []
26 }
27
28 for endpoint, metric in metrics.items():
29 result = {
30 'status': 'pass',
31 'metrics': {},
32 'issues': []
33 }
34
35 # Check response time
36 if metric.avg_response_time > self.thresholds['response_time_ms']:
37 result['status'] = 'fail'
38 result['issues'].append(
39 f"Avg response time {metric.avg_response_time:.2f}ms exceeds "
40 f"threshold {self.thresholds['response_time_ms']}ms"
41 )
42 analysis['violations'].append({
43 'endpoint': endpoint,
44 'metric': 'response_time',
45 'value': metric.avg_response_time,
46 'threshold': self.thresholds['response_time_ms']
47 })
48
49 # Check error rate
50 error_rate_percent = metric.failure_rate * 100
51 if error_rate_percent > self.thresholds['error_rate_percent']:
52 result['status'] = 'fail'
53 result['issues'].append(
54 f"Error rate {error_rate_percent:.2f}% exceeds "
55 f"threshold {self.thresholds['error_rate_percent']}%"
56 )
57
58 # Check throughput
59 if metric.requests_per_second < self.thresholds['throughput_rps']:
60 result['status'] = 'fail'
61 result['issues'].append(
62 f"Throughput {metric.requests_per_second:.2f} RPS below "
63 f"threshold {self.thresholds['throughput_rps']} RPS"
64 )
65
66 result['metrics'] = {
67 'avg_response_ms': metric.avg_response_time,
68 'p95_response_ms': metric.percentile_95,
69 'p99_response_ms': metric.percentile_99,
70 'throughput_rps': metric.requests_per_second,
71 'error_rate_percent': error_rate_percent
72 }
73
74 analysis['endpoint_results'][endpoint] = result
75
76 if result['status'] == 'fail':
77 analysis['overall_status'] = 'fail'
78
79 return analysis
80
81# Example: Analyze results
82analyzer = PerformanceAnalyzer()
83analysis = analyzer.analyze_results(metrics)
84
85print(f"\nš Performance Analysis: {analysis['overall_status'].upper()}")
86print(f"\nViolations: {len(analysis['violations'])}")
87for violation in analysis['violations']:
88 print(f" - {violation['endpoint']}: {violation['metric']} = {violation['value']:.2f}")Key Load Testing Patterns:
Profiling identifies performance bottlenecks by measuring code execution time, memory usage, and resource consumption. Python provides powerful profiling tools for finding optimization opportunities.
CPU Profiling with cProfile:
1import cProfile
2import pstats
3import io
4from pstats import SortKey
5from typing import Dict, List
6import time
7
8class PerformanceProfiler:
9 """
10 Profile application performance and identify bottlenecks
11 """
12
13 def __init__(self):
14 self.profiler = cProfile.Profile()
15 self.profile_data = None
16
17 def profile_function(self, func, *args, **kwargs):
18 """
19 Profile a specific function
20
21 Args:
22 func: Function to profile
23 *args, **kwargs: Function arguments
24
25 Returns:
26 Function result and profile stats
27 """
28 self.profiler.enable()
29 result = func(*args, **kwargs)
30 self.profiler.disable()
31
32 # Get stats
33 s = io.StringIO()
34 ps = pstats.Stats(self.profiler, stream=s).sort_stats(SortKey.CUMULATIVE)
35 ps.print_stats(20) # Top 20 functions
36
37 profile_output = s.getvalue()
38
39 return result, profile_output
40
41 def find_bottlenecks(self, threshold_ms: float = 100) -> List[Dict]:
42 """
43 Identify performance bottlenecks
44
45 Args:
46 threshold_ms: Threshold in milliseconds
47
48 Returns:
49 List of bottlenecks
50 """
51 bottlenecks = []
52
53 s = io.StringIO()
54 ps = pstats.Stats(self.profiler, stream=s)
55
56 # Analyze stats
57 for func, stats in ps.stats.items():
58 filename, line, func_name = func
59 cumtime = stats[3] # Cumulative time
60
61 cumtime_ms = cumtime * 1000
62 if cumtime_ms > threshold_ms:
63 bottlenecks.append({
64 'function': func_name,
65 'file': filename,
66 'line': line,
67 'cumulative_time_ms': cumtime_ms,
68 'calls': stats[0]
69 })
70
71 # Sort by cumulative time
72 bottlenecks.sort(key=lambda x: x['cumulative_time_ms'], reverse=True)
73
74 return bottlenecks
75
76# Example function to profile
77def slow_function():
78 """Simulated slow function"""
79 total = 0
80 for i in range(1000000):
81 total += i * i
82 return total
83
84def database_query_simulation():
85 """Simulated database query"""
86 time.sleep(0.5) # Simulate I/O wait
87 return [{"id": i, "data": f"record_{i}"} for i in range(1000)]
88
89# Profile execution
90profiler = PerformanceProfiler()
91result, profile_output = profiler.profile_function(slow_function)
92
93print("š Profile Results:")
94print(profile_output)
95
96bottlenecks = profiler.find_bottlenecks(threshold_ms=50)
97print(f"\nš Identified {len(bottlenecks)} bottlenecks:")
98for bn in bottlenecks[:5]:
99 print(f" - {bn['function']}: {bn['cumulative_time_ms']:.2f}ms ({bn['calls']} calls)")Memory Profiling:
1import tracemalloc
2from dataclasses import dataclass
3from typing import List, Tuple
4
5@dataclass
6class MemorySnapshot:
7 """Memory usage snapshot"""
8 timestamp: str
9 current_mb: float
10 peak_mb: float
11 top_allocations: List[Tuple[str, int, float]]
12
13class MemoryProfiler:
14 """
15 Profile memory usage and detect memory leaks
16 """
17
18 def __init__(self):
19 self.snapshots = []
20
21 def start(self):
22 """Start memory profiling"""
23 tracemalloc.start()
24 logger.info("š Memory profiling started")
25
26 def take_snapshot(self) -> MemorySnapshot:
27 """
28 Take memory snapshot
29
30 Returns:
31 Memory snapshot with current usage and top allocations
32 """
33 current, peak = tracemalloc.get_traced_memory()
34
35 # Get top memory consumers
36 snapshot = tracemalloc.take_snapshot()
37 top_stats = snapshot.statistics('lineno')
38
39 top_allocations = []
40 for stat in top_stats[:10]:
41 top_allocations.append((
42 str(stat),
43 stat.count,
44 stat.size / 1024 / 1024 # MB
45 ))
46
47 mem_snapshot = MemorySnapshot(
48 timestamp=datetime.now().isoformat(),
49 current_mb=current / 1024 / 1024,
50 peak_mb=peak / 1024 / 1024,
51 top_allocations=top_allocations
52 )
53
54 self.snapshots.append(mem_snapshot)
55 return mem_snapshot
56
57 def detect_memory_leak(self) -> bool:
58 """
59 Detect potential memory leak
60
61 Returns:
62 True if memory leak detected
63 """
64 if len(self.snapshots) < 3:
65 return False
66
67 # Check if memory consistently increases
68 recent = self.snapshots[-3:]
69 memory_values = [s.current_mb for s in recent]
70
71 # Simple leak detection: memory increases in all samples
72 is_increasing = all(
73 memory_values[i] < memory_values[i+1]
74 for i in range(len(memory_values)-1)
75 )
76
77 if is_increasing:
78 increase = memory_values[-1] - memory_values[0]
79 logger.warning(f"ā ļø Potential memory leak: +{increase:.2f}MB")
80 return True
81
82 return False
83
84 def stop(self):
85 """Stop memory profiling"""
86 tracemalloc.stop()
87 logger.info("ā
Memory profiling stopped")
88
89# Example usage
90mem_profiler = MemoryProfiler()
91mem_profiler.start()
92
93# Simulate workload
94data = []
95for i in range(3):
96 # Allocate memory
97 data.extend([{"id": j, "data": "x" * 1000} for j in range(10000)])
98
99 # Take snapshot
100 snapshot = mem_profiler.take_snapshot()
101 print(f"\nSnapshot {i+1}: {snapshot.current_mb:.2f}MB (Peak: {snapshot.peak_mb:.2f}MB)")
102
103 # Check for leak
104 if mem_profiler.detect_memory_leak():
105 print("šØ Memory leak detected!")
106
107mem_profiler.stop()Database Query Performance Analysis:
1import time
2from functools import wraps
3from typing import Callable, Dict
4from collections import defaultdict
5
6class QueryPerformanceMonitor:
7 """Monitor database query performance"""
8
9 def __init__(self):
10 self.query_stats = defaultdict(lambda: {
11 'count': 0,
12 'total_time': 0,
13 'slow_queries': []
14 })
15 self.slow_query_threshold_ms = 100
16
17 def monitor_query(self, query_name: str):
18 """Decorator to monitor query performance"""
19 def decorator(func: Callable):
20 @wraps(func)
21 def wrapper(*args, **kwargs):
22 start = time.time()
23 result = func(*args, **kwargs)
24 duration = (time.time() - start) * 1000 # ms
25
26 # Record stats
27 stats = self.query_stats[query_name]
28 stats['count'] += 1
29 stats['total_time'] += duration
30
31 # Track slow queries
32 if duration > self.slow_query_threshold_ms:
33 stats['slow_queries'].append({
34 'duration_ms': duration,
35 'timestamp': datetime.now().isoformat(),
36 'args': str(args)[:100]
37 })
38 logger.warning(f"š Slow query: {query_name} ({duration:.2f}ms)")
39
40 return result
41 return wrapper
42 return decorator
43
44 def get_performance_report(self) -> Dict:
45 """Generate performance report"""
46 report = {
47 'total_queries': sum(s['count'] for s in self.query_stats.values()),
48 'queries': []
49 }
50
51 for query_name, stats in self.query_stats.items():
52 avg_time = stats['total_time'] / stats['count'] if stats['count'] > 0 else 0
53
54 report['queries'].append({
55 'name': query_name,
56 'count': stats['count'],
57 'avg_time_ms': avg_time,
58 'total_time_ms': stats['total_time'],
59 'slow_count': len(stats['slow_queries'])
60 })
61
62 # Sort by total time
63 report['queries'].sort(key=lambda x: x['total_time_ms'], reverse=True)
64
65 return report
66
67# Example usage
68monitor = QueryPerformanceMonitor()
69
70@monitor.monitor_query("get_user_by_id")
71def get_user(user_id: int):
72 time.sleep(0.05) # Simulate query
73 return {"id": user_id, "name": f"User {user_id}"}
74
75@monitor.monitor_query("get_all_users")
76def get_all_users():
77 time.sleep(0.15) # Simulate slow query
78 return [{"id": i, "name": f"User {i}"} for i in range(100)]
79
80# Execute queries
81for i in range(50):
82 get_user(i)
83 if i % 10 == 0:
84 get_all_users()
85
86# Get report
87report = monitor.get_performance_report()
88print(f"\nš Query Performance Report:")
89print(f"Total Queries: {report['total_queries']}")
90print(f"\nTop Queries by Time:")
91for query in report['queries'][:5]:
92 print(f" - {query['name']}: {query['avg_time_ms']:.2f}ms avg "
93 f"({query['count']} calls, {query['slow_count']} slow)")Performance Profiling Best Practices:
Continuous performance monitoring tracks metrics over time, detects regressions automatically, and enables data-driven optimization decisions.
Automated Performance Regression Detection:
1from datetime import datetime, timedelta
2from typing import List, Optional
3import json
4from dataclasses import dataclass, asdict
5
6@dataclass
7class PerformanceBenchmark:
8 """Performance benchmark result"""
9 timestamp: str
10 endpoint: str
11 avg_response_ms: float
12 p95_response_ms: float
13 throughput_rps: float
14 error_rate: float
15
16class PerformanceRegressionDetector:
17 """
18 Detect performance regressions automatically
19 """
20
21 def __init__(self, baseline_file: str = 'baseline.json'):
22 self.baseline_file = baseline_file
23 self.baseline = self._load_baseline()
24 self.regression_threshold = 0.20 # 20% degradation
25
26 def _load_baseline(self) -> Dict[str, PerformanceBenchmark]:
27 """Load baseline performance metrics"""
28 try:
29 with open(self.baseline_file, 'r') as f:
30 data = json.load(f)
31 return {
32 k: PerformanceBenchmark(**v)
33 for k, v in data.items()
34 }
35 except FileNotFoundError:
36 logger.info("No baseline found, will create new baseline")
37 return {}
38
39 def save_baseline(self, benchmarks: Dict[str, PerformanceBenchmark]):
40 """Save current results as baseline"""
41 data = {k: asdict(v) for k, v in benchmarks.items()}
42 with open(self.baseline_file, 'w') as f:
43 json.dump(data, f, indent=2)
44 logger.info(f"ā
Baseline saved: {len(benchmarks)} endpoints")
45
46 def detect_regressions(
47 self,
48 current: Dict[str, PerformanceBenchmark]
49 ) -> List[Dict]:
50 """
51 Compare current performance against baseline
52
53 Args:
54 current: Current benchmark results
55
56 Returns:
57 List of detected regressions
58 """
59 regressions = []
60
61 for endpoint, current_bench in current.items():
62 if endpoint not in self.baseline:
63 logger.info(f"New endpoint (no baseline): {endpoint}")
64 continue
65
66 baseline_bench = self.baseline[endpoint]
67
68 # Check response time regression
69 response_degradation = (
70 (current_bench.avg_response_ms - baseline_bench.avg_response_ms) /
71 baseline_bench.avg_response_ms
72 )
73
74 if response_degradation > self.regression_threshold:
75 regressions.append({
76 'endpoint': endpoint,
77 'metric': 'avg_response_time',
78 'baseline_ms': baseline_bench.avg_response_ms,
79 'current_ms': current_bench.avg_response_ms,
80 'degradation_percent': response_degradation * 100,
81 'severity': 'high' if response_degradation > 0.5 else 'medium'
82 })
83
84 # Check throughput regression
85 throughput_degradation = (
86 (baseline_bench.throughput_rps - current_bench.throughput_rps) /
87 baseline_bench.throughput_rps
88 )
89
90 if throughput_degradation > self.regression_threshold:
91 regressions.append({
92 'endpoint': endpoint,
93 'metric': 'throughput',
94 'baseline_rps': baseline_bench.throughput_rps,
95 'current_rps': current_bench.throughput_rps,
96 'degradation_percent': throughput_degradation * 100,
97 'severity': 'high' if throughput_degradation > 0.5 else 'medium'
98 })
99
100 if regressions:
101 logger.warning(f"šØ {len(regressions)} performance regression(s) detected!")
102 else:
103 logger.info("ā
No performance regressions detected")
104
105 return regressions
106
107# Example: Detect regressions
108detector = PerformanceRegressionDetector()
109
110# Current test results
111current_results = {
112 'GET /api/users': PerformanceBenchmark(
113 timestamp=datetime.now().isoformat(),
114 endpoint='GET /api/users',
115 avg_response_ms=250, # Regressed from baseline
116 p95_response_ms=450,
117 throughput_rps=150,
118 error_rate=0.01
119 ),
120 'POST /api/users': PerformanceBenchmark(
121 timestamp=datetime.now().isoformat(),
122 endpoint='POST /api/users',
123 avg_response_ms=180,
124 p95_response_ms=320,
125 throughput_rps=80,
126 error_rate=0.02
127 )
128}
129
130# Check for regressions
131regressions = detector.detect_regressions(current_results)
132
133if regressions:
134 print("\nšØ Performance Regressions Detected:")
135 for reg in regressions:
136 print(f"\n [{reg['severity'].upper()}] {reg['endpoint']} - {reg['metric']}")
137 print(f" Baseline: {reg.get('baseline_ms', reg.get('baseline_rps')):.2f}")
138 print(f" Current: {reg.get('current_ms', reg.get('current_rps')):.2f}")
139 print(f" Degradation: {reg['degradation_percent']:.1f}%")
140else:
141 # No regressions, save as new baseline
142 detector.save_baseline(current_results)Continuous Performance Monitoring:
1from prometheus_client import Counter, Histogram, Gauge, CollectorRegistry
2import time
3
4class PerformanceMonitor:
5 """
6 Real-time performance monitoring with Prometheus metrics
7 """
8
9 def __init__(self):
10 self.registry = CollectorRegistry()
11
12 # Request metrics
13 self.request_duration = Histogram(
14 'http_request_duration_seconds',
15 'HTTP request duration',
16 ['method', 'endpoint', 'status'],
17 buckets=[0.01, 0.05, 0.1, 0.5, 1.0, 5.0],
18 registry=self.registry
19 )
20
21 self.request_counter = Counter(
22 'http_requests_total',
23 'Total HTTP requests',
24 ['method', 'endpoint', 'status'],
25 registry=self.registry
26 )
27
28 # Performance metrics
29 self.active_requests = Gauge(
30 'http_active_requests',
31 'Currently active requests',
32 registry=self.registry
33 )
34
35 self.db_query_duration = Histogram(
36 'db_query_duration_seconds',
37 'Database query duration',
38 ['query_type'],
39 buckets=[0.001, 0.01, 0.05, 0.1, 0.5, 1.0],
40 registry=self.registry
41 )
42
43 def track_request(self, method: str, endpoint: str):
44 """Context manager to track request performance"""
45 class RequestTracker:
46 def __init__(self, monitor, method, endpoint):
47 self.monitor = monitor
48 self.method = method
49 self.endpoint = endpoint
50 self.start_time = None
51
52 def __enter__(self):
53 self.start_time = time.time()
54 self.monitor.active_requests.inc()
55 return self
56
57 def __exit__(self, exc_type, exc_val, exc_tb):
58 duration = time.time() - self.start_time
59 status = '500' if exc_type else '200'
60
61 self.monitor.request_duration.labels(
62 method=self.method,
63 endpoint=self.endpoint,
64 status=status
65 ).observe(duration)
66
67 self.monitor.request_counter.labels(
68 method=self.method,
69 endpoint=self.endpoint,
70 status=status
71 ).inc()
72
73 self.monitor.active_requests.dec()
74
75 return RequestTracker(self, method, endpoint)
76
77 def track_db_query(self, query_type: str):
78 """Context manager to track database query performance"""
79 class QueryTracker:
80 def __init__(self, monitor, query_type):
81 self.monitor = monitor
82 self.query_type = query_type
83 self.start_time = None
84
85 def __enter__(self):
86 self.start_time = time.time()
87 return self
88
89 def __exit__(self, exc_type, exc_val, exc_tb):
90 duration = time.time() - self.start_time
91 self.monitor.db_query_duration.labels(
92 query_type=self.query_type
93 ).observe(duration)
94
95 return QueryTracker(self, query_type)
96
97# Example usage
98monitor = PerformanceMonitor()
99
100def handle_api_request(user_id: int):
101 """Example API handler with monitoring"""
102 with monitor.track_request('GET', '/api/users/:id'):
103 # Database query
104 with monitor.track_db_query('select'):
105 time.sleep(0.05) # Simulate query
106 user_data = {"id": user_id, "name": f"User {user_id}"}
107
108 # Processing
109 time.sleep(0.02) # Simulate processing
110
111 return user_data
112
113# Simulate requests
114for i in range(100):
115 handle_api_request(i)
116
117print("š Performance metrics collected and exposed for Prometheus")Performance Optimization Automation:
1class PerformanceOptimizer:
2 """
3 Automated performance optimization
4 """
5
6 def __init__(self):
7 self.optimization_rules = []
8
9 def analyze_and_optimize(self, metrics: Dict) -> List[Dict]:
10 """
11 Analyze metrics and suggest optimizations
12
13 Returns:
14 List of optimization recommendations
15 """
16 recommendations = []
17
18 for endpoint, data in metrics.items():
19 # High response time ā Consider caching
20 if data['avg_response_ms'] > 500:
21 recommendations.append({
22 'endpoint': endpoint,
23 'issue': 'High response time',
24 'optimization': 'Implement caching (Redis/Memcached)',
25 'expected_improvement': '50-80% reduction',
26 'priority': 'high'
27 })
28
29 # High database query time ā Add indexes
30 if data.get('db_time_ms', 0) > data['avg_response_ms'] * 0.6:
31 recommendations.append({
32 'endpoint': endpoint,
33 'issue': 'Database queries consuming >60% of response time',
34 'optimization': 'Add database indexes or optimize queries',
35 'expected_improvement': '40-60% reduction',
36 'priority': 'high'
37 })
38
39 # Low throughput ā Scale horizontally
40 if data.get('throughput_rps', 0) < 50:
41 recommendations.append({
42 'endpoint': endpoint,
43 'issue': 'Low throughput',
44 'optimization': 'Scale application horizontally (add replicas)',
45 'expected_improvement': 'Linear scaling',
46 'priority': 'medium'
47 })
48
49 return recommendations
50
51# Example: Get optimization recommendations
52optimizer = PerformanceOptimizer()
53
54metrics = {
55 'GET /api/reports': {
56 'avg_response_ms': 850,
57 'db_time_ms': 600,
58 'throughput_rps': 35
59 }
60}
61
62recommendations = optimizer.analyze_and_optimize(metrics)
63
64print("\nš§ Performance Optimization Recommendations:")
65for rec in recommendations:
66 print(f"\n [{rec['priority'].upper()}] {rec['endpoint']}")
67 print(f" Issue: {rec['issue']}")
68 print(f" Recommendation: {rec['optimization']}")
69 print(f" Expected: {rec['expected_improvement']}")Continuous Monitoring Best Practices:
Build a production-ready performance testing platform with load testing, profiling, regression detection, and automated optimization recommendations.
1#!/usr/bin/env python3
2"""
3ā” Production Performance Testing and Optimization Platform
4Topic: Load Testing + Profiling + Regression Detection + Auto-Optimization
5Author: Performance Engineering Team
6Dependencies: pip install locust requests prometheus-client
7"""
8
9import time
10import cProfile
11import pstats
12import io
13import json
14import requests
15from datetime import datetime
16from typing import Dict, List, Optional, Tuple
17from dataclasses import dataclass, asdict
18from collections import defaultdict
19from enum import Enum
20import logging
21import sys
22
23# Configure logging
24logging.basicConfig(
25 level=logging.INFO,
26 format='%(asctime)s - %(levelname)s - %(message)s',
27 handlers=[
28 logging.FileHandler('performance-testing.log'),
29 logging.StreamHandler(sys.stdout)
30 ]
31)
32logger = logging.getLogger(__name__)
33
34
35class TestStatus(Enum):
36 """Performance test status"""
37 PASS = "pass"
38 FAIL = "fail"
39 WARNING = "warning"
40
41
42@dataclass
43class PerformanceMetrics:
44 """Performance test metrics"""
45 endpoint: str
46 avg_response_ms: float
47 p95_response_ms: float
48 p99_response_ms: float
49 min_response_ms: float
50 max_response_ms: float
51 throughput_rps: float
52 error_rate: float
53 total_requests: int
54
55
56@dataclass
57class PerformanceBenchmark:
58 """Performance benchmark for regression detection"""
59 timestamp: str
60 endpoint: str
61 avg_response_ms: float
62 p95_response_ms: float
63 throughput_rps: float
64 error_rate: float
65
66
67class SimpleLoadTester:
68 """
69 Simple HTTP load tester
70 """
71
72 def __init__(self, base_url: str):
73 self.base_url = base_url
74 self.results = defaultdict(list)
75
76 def run_load_test(
77 self,
78 endpoint: str,
79 num_requests: int = 100,
80 concurrent_users: int = 10
81 ) -> PerformanceMetrics:
82 """
83 Run simple load test on endpoint
84
85 Args:
86 endpoint: API endpoint to test
87 num_requests: Total number of requests
88 concurrent_users: Number of concurrent virtual users
89
90 Returns:
91 Performance metrics
92 """
93 logger.info(f"š Load testing {endpoint}: {num_requests} requests, {concurrent_users} users")
94
95 response_times = []
96 errors = 0
97 start_time = time.time()
98
99 # Simple sequential load test (production would use threading/asyncio)
100 requests_per_user = num_requests // concurrent_users
101
102 for i in range(num_requests):
103 req_start = time.time()
104 try:
105 response = requests.get(
106 f"{self.base_url}{endpoint}",
107 timeout=30
108 )
109 req_duration = (time.time() - req_start) * 1000 # ms
110
111 response_times.append(req_duration)
112
113 if response.status_code >= 400:
114 errors += 1
115
116 except requests.RequestException as e:
117 errors += 1
118 logger.warning(f"Request failed: {e}")
119
120 total_duration = time.time() - start_time
121
122 # Calculate metrics
123 response_times.sort()
124 p95_index = int(len(response_times) * 0.95)
125 p99_index = int(len(response_times) * 0.99)
126
127 metrics = PerformanceMetrics(
128 endpoint=endpoint,
129 avg_response_ms=sum(response_times) / len(response_times) if response_times else 0,
130 p95_response_ms=response_times[p95_index] if response_times else 0,
131 p99_response_ms=response_times[p99_index] if response_times else 0,
132 min_response_ms=min(response_times) if response_times else 0,
133 max_response_ms=max(response_times) if response_times else 0,
134 throughput_rps=num_requests / total_duration,
135 error_rate=errors / num_requests,
136 total_requests=num_requests
137 )
138
139 return metrics
140
141
142class PerformanceAnalyzer:
143 """Analyze performance test results"""
144
145 def __init__(self):
146 self.thresholds = {
147 'response_time_ms': 200,
148 'p95_response_ms': 500,
149 'error_rate_percent': 1.0,
150 'throughput_rps': 50
151 }
152
153 def analyze_metrics(self, metrics: PerformanceMetrics) -> Dict:
154 """
155 Analyze metrics against thresholds
156
157 Returns:
158 Analysis result with status and issues
159 """
160 result = {
161 'endpoint': metrics.endpoint,
162 'status': TestStatus.PASS.value,
163 'issues': [],
164 'metrics': asdict(metrics)
165 }
166
167 # Check response time
168 if metrics.avg_response_ms > self.thresholds['response_time_ms']:
169 result['status'] = TestStatus.FAIL.value
170 result['issues'].append(
171 f"Avg response time {metrics.avg_response_ms:.2f}ms exceeds "
172 f"threshold {self.thresholds['response_time_ms']}ms"
173 )
174
175 # Check p95
176 if metrics.p95_response_ms > self.thresholds['p95_response_ms']:
177 result['status'] = TestStatus.FAIL.value
178 result['issues'].append(
179 f"P95 response time {metrics.p95_response_ms:.2f}ms exceeds "
180 f"threshold {self.thresholds['p95_response_ms']}ms"
181 )
182
183 # Check error rate
184 error_rate_percent = metrics.error_rate * 100
185 if error_rate_percent > self.thresholds['error_rate_percent']:
186 result['status'] = TestStatus.FAIL.value
187 result['issues'].append(
188 f"Error rate {error_rate_percent:.2f}% exceeds "
189 f"threshold {self.thresholds['error_rate_percent']}%"
190 )
191
192 # Check throughput
193 if metrics.throughput_rps < self.thresholds['throughput_rps']:
194 result['status'] = TestStatus.WARNING.value
195 result['issues'].append(
196 f"Throughput {metrics.throughput_rps:.2f} RPS below "
197 f"threshold {self.thresholds['throughput_rps']} RPS"
198 )
199
200 return result
201
202
203class RegressionDetector:
204 """Detect performance regressions"""
205
206 def __init__(self, baseline_file: str = 'baseline.json'):
207 self.baseline_file = baseline_file
208 self.baseline = self._load_baseline()
209 self.regression_threshold = 0.20 # 20% degradation
210
211 def _load_baseline(self) -> Dict[str, PerformanceBenchmark]:
212 """Load baseline metrics"""
213 try:
214 with open(self.baseline_file, 'r') as f:
215 data = json.load(f)
216 return {
217 k: PerformanceBenchmark(**v)
218 for k, v in data.items()
219 }
220 except FileNotFoundError:
221 logger.info("No baseline found")
222 return {}
223
224 def save_baseline(self, benchmarks: Dict[str, PerformanceBenchmark]):
225 """Save baseline"""
226 data = {k: asdict(v) for k, v in benchmarks.items()}
227 with open(self.baseline_file, 'w') as f:
228 json.dump(data, f, indent=2)
229 logger.info(f"ā
Baseline saved")
230
231 def detect_regressions(
232 self,
233 current: Dict[str, PerformanceBenchmark]
234 ) -> List[Dict]:
235 """Detect regressions"""
236 regressions = []
237
238 for endpoint, current_bench in current.items():
239 if endpoint not in self.baseline:
240 continue
241
242 baseline_bench = self.baseline[endpoint]
243
244 # Check response time regression
245 response_degradation = (
246 (current_bench.avg_response_ms - baseline_bench.avg_response_ms) /
247 baseline_bench.avg_response_ms
248 )
249
250 if response_degradation > self.regression_threshold:
251 regressions.append({
252 'endpoint': endpoint,
253 'metric': 'avg_response_time',
254 'baseline_ms': baseline_bench.avg_response_ms,
255 'current_ms': current_bench.avg_response_ms,
256 'degradation_percent': response_degradation * 100
257 })
258
259 return regressions
260
261
262class PerformanceOptimizer:
263 """Automated performance optimization recommendations"""
264
265 def analyze_and_recommend(self, metrics: PerformanceMetrics) -> List[Dict]:
266 """Generate optimization recommendations"""
267 recommendations = []
268
269 # High response time ā Caching
270 if metrics.avg_response_ms > 500:
271 recommendations.append({
272 'priority': 'high',
273 'issue': f'High response time: {metrics.avg_response_ms:.2f}ms',
274 'recommendation': 'Implement caching (Redis/Memcached)',
275 'expected_improvement': '50-80% reduction'
276 })
277
278 # High p99 ā Optimize slow paths
279 if metrics.p99_response_ms > metrics.avg_response_ms * 3:
280 recommendations.append({
281 'priority': 'medium',
282 'issue': f'High P99: {metrics.p99_response_ms:.2f}ms',
283 'recommendation': 'Profile and optimize slow code paths',
284 'expected_improvement': '40-60% reduction in tail latency'
285 })
286
287 # Low throughput ā Scale
288 if metrics.throughput_rps < 100:
289 recommendations.append({
290 'priority': 'medium',
291 'issue': f'Low throughput: {metrics.throughput_rps:.2f} RPS',
292 'recommendation': 'Scale horizontally (add replicas)',
293 'expected_improvement': 'Linear scaling'
294 })
295
296 return recommendations
297
298
299class PerformanceTestingPlatform:
300 """
301 Complete performance testing and optimization platform
302 """
303
304 def __init__(self, base_url: str = "http://localhost:8000"):
305 logger.info("š Initializing Performance Testing Platform")
306
307 self.base_url = base_url
308 self.load_tester = SimpleLoadTester(base_url)
309 self.analyzer = PerformanceAnalyzer()
310 self.regression_detector = RegressionDetector()
311 self.optimizer = PerformanceOptimizer()
312
313 logger.info("ā
Platform initialized")
314
315 def run_comprehensive_test(self, endpoints: List[str]) -> Dict:
316 """
317 Run comprehensive performance test
318
319 Args:
320 endpoints: List of endpoints to test
321
322 Returns:
323 Complete test report
324 """
325 logger.info("\n" + "="*70)
326 logger.info("ā” RUNNING PERFORMANCE TEST SUITE")
327 logger.info("="*70)
328
329 report = {
330 'timestamp': datetime.now().isoformat(),
331 'base_url': self.base_url,
332 'overall_status': TestStatus.PASS.value,
333 'endpoint_results': {},
334 'regressions': [],
335 'recommendations': []
336 }
337
338 # Test each endpoint
339 for endpoint in endpoints:
340 logger.info(f"\nš Testing: {endpoint}")
341
342 # Run load test
343 metrics = self.load_tester.run_load_test(
344 endpoint,
345 num_requests=100,
346 concurrent_users=10
347 )
348
349 # Analyze results
350 analysis = self.analyzer.analyze_metrics(metrics)
351 report['endpoint_results'][endpoint] = analysis
352
353 if analysis['status'] != TestStatus.PASS.value:
354 report['overall_status'] = TestStatus.FAIL.value
355
356 # Get optimization recommendations
357 recommendations = self.optimizer.analyze_and_recommend(metrics)
358 if recommendations:
359 report['recommendations'].extend([
360 {'endpoint': endpoint, **rec}
361 for rec in recommendations
362 ])
363
364 # Print results
365 logger.info(f" Status: {analysis['status'].upper()}")
366 logger.info(f" Avg: {metrics.avg_response_ms:.2f}ms")
367 logger.info(f" P95: {metrics.p95_response_ms:.2f}ms")
368 logger.info(f" RPS: {metrics.throughput_rps:.2f}")
369
370 # Check for regressions
371 current_benchmarks = {
372 endpoint: PerformanceBenchmark(
373 timestamp=datetime.now().isoformat(),
374 endpoint=endpoint,
375 avg_response_ms=result['metrics']['avg_response_ms'],
376 p95_response_ms=result['metrics']['p95_response_ms'],
377 throughput_rps=result['metrics']['throughput_rps'],
378 error_rate=result['metrics']['error_rate']
379 )
380 for endpoint, result in report['endpoint_results'].items()
381 }
382
383 regressions = self.regression_detector.detect_regressions(current_benchmarks)
384 report['regressions'] = regressions
385
386 if not regressions and report['overall_status'] == TestStatus.PASS.value:
387 self.regression_detector.save_baseline(current_benchmarks)
388
389 return report
390
391
392def main():
393 """
394 Main entry point - Performance testing demonstration
395 """
396 print("\n" + "="*70)
397 print("ā” Production Performance Testing Platform")
398 print("="*70 + "\n")
399
400 try:
401 # Initialize platform (demo mode - no actual server)
402 platform = PerformanceTestingPlatform(base_url="https://api.example.com")
403
404 # Demo: Simulate test results
405 demo_metrics = PerformanceMetrics(
406 endpoint="/api/users",
407 avg_response_ms=150,
408 p95_response_ms=280,
409 p99_response_ms=450,
410 min_response_ms=45,
411 max_response_ms=650,
412 throughput_rps=125,
413 error_rate=0.005,
414 total_requests=1000
415 )
416
417 # Analyze
418 analyzer = PerformanceAnalyzer()
419 analysis = analyzer.analyze_metrics(demo_metrics)
420
421 print("\nš PERFORMANCE TEST RESULTS:")
422 print(f" Endpoint: {demo_metrics.endpoint}")
423 print(f" Status: {analysis['status'].upper()}")
424 print(f"\n Response Times:")
425 print(f" Avg: {demo_metrics.avg_response_ms:.2f}ms")
426 print(f" P95: {demo_metrics.p95_response_ms:.2f}ms")
427 print(f" P99: {demo_metrics.p99_response_ms:.2f}ms")
428 print(f"\n Performance:")
429 print(f" Throughput: {demo_metrics.throughput_rps:.2f} RPS")
430 print(f" Error Rate: {demo_metrics.error_rate*100:.2f}%")
431
432 if analysis['issues']:
433 print(f"\n ā ļø Issues Found:")
434 for issue in analysis['issues']:
435 print(f" - {issue}")
436
437 # Get optimization recommendations
438 optimizer = PerformanceOptimizer()
439 recommendations = optimizer.analyze_and_recommend(demo_metrics)
440
441 if recommendations:
442 print(f"\nš§ OPTIMIZATION RECOMMENDATIONS:")
443 for rec in recommendations:
444 print(f"\n [{rec['priority'].upper()}]")
445 print(f" Issue: {rec['issue']}")
446 print(f" Fix: {rec['recommendation']}")
447 print(f" Expected: {rec['expected_improvement']}")
448
449 print("\n" + "="*70)
450 logger.info("\nš Performance testing completed!")
451
452 except KeyboardInterrupt:
453 logger.info("\nā ļø Interrupted by user")
454 sys.exit(130)
455 except Exception as e:
456 logger.error(f"\nā Fatal error: {e}")
457 import traceback
458 traceback.print_exc()
459 sys.exit(1)
460
461
462if __name__ == "__main__":
463 main()š Code Breakdown:
1. Professional Structure:
2. Production-Ready Features:
3. Performance Testing Specific Features:
4. DevOps Best Practices:
This implementation demonstrates production-ready performance testing automation!
Your progress is saved in this browser, on this device only.