Deploy AI features to production safely - implement error handling, fallback mechanisms, performance monitoring, A/B testing, incident response, and production best practices.
Deploying AI to production is fundamentally different from traditional software - AI systems can fail in unpredictable ways, API costs can spike unexpectedly, and model behavior can drift over time. Production AI requires robust error handling, comprehensive monitoring, and careful management.
The stakes are high: a single API outage can cost thousands of dollars, poor error handling can break user experiences, and unmonitored systems can degrade silently. Yet with proper architecture and monitoring, production AI can be reliable, cost-effective, and continuously improving.
In this lesson, you'll learn production deployment best practices from real-world AI applications. You'll implement bullet-proof error handling, set up monitoring dashboards, A/B test features, and build systems that gracefully handle the inevitable failures.
Design resilient AI systems that handle failures gracefully and scale efficiently.
Production Architecture Patterns:
1// Production-Ready AI Service Architecture
2
3class ProductionAIService {
4 constructor() {
5 this.primaryProvider = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
6 this.fallbackProvider = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
7 this.cache = new Redis(process.env.REDIS_URL);
8 this.queue = new BullQueue('ai-requests');
9 this.metrics = new MetricsCollector();
10 this.logger = new Logger('ai-service');
11 }
12
13 // Request with full production safeguards
14 async processRequest(request) {
15 const requestId = this.generateRequestId();
16 const startTime = Date.now();
17
18 try {
19 // 1. Validate input
20 this.validateRequest(request);
21
22 // 2. Check cache
23 const cached = await this.checkCache(request);
24 if (cached) {
25 this.metrics.recordCacheHit();
26 return this.formatResponse(cached, { cached: true });
27 }
28
29 // 3. Rate limit check
30 await this.checkRateLimit(request.userId);
31
32 // 4. Execute with retry and fallback
33 const result = await this.executeWithFallback(request);
34
35 // 5. Cache result
36 await this.cacheResult(request, result);
37
38 // 6. Record metrics
39 const duration = Date.now() - startTime;
40 this.metrics.recordRequest({
41 requestId,
42 duration,
43 model: result.model,
44 tokens: result.usage,
45 cost: result.cost,
46 success: true
47 });
48
49 return this.formatResponse(result);
50
51 } catch (error) {
52 // Error handling
53 const duration = Date.now() - startTime;
54
55 this.logger.error('Request failed', {
56 requestId,
57 error: error.message,
58 duration
59 });
60
61 this.metrics.recordError({
62 requestId,
63 errorType: error.constructor.name,
64 duration
65 });
66
67 // Return graceful error
68 return this.handleError(error, request);
69 }
70 }
71
72 // Primary request with automatic fallback
73 async executeWithFallback(request) {
74 try {
75 // Try primary provider
76 return await this.executeWithRetry(
77 () => this.callPrimaryProvider(request),
78 { maxRetries: 3, backoff: 'exponential' }
79 );
80
81 } catch (primaryError) {
82 this.logger.warn('Primary provider failed, falling back', {
83 error: primaryError.message
84 });
85
86 // Try fallback provider
87 try {
88 return await this.callFallbackProvider(request);
89
90 } catch (fallbackError) {
91 this.logger.error('All providers failed', {
92 primaryError: primaryError.message,
93 fallbackError: fallbackError.message
94 });
95
96 // Return cached or default response
97 return await this.getEmergencyResponse(request);
98 }
99 }
100 }
101
102 // Retry with exponential backoff
103 async executeWithRetry(fn, options = {}) {
104 const maxRetries = options.maxRetries || 3;
105 const backoff = options.backoff || 'exponential';
106
107 for (let attempt = 1; attempt <= maxRetries; attempt++) {
108 try {
109 return await fn();
110
111 } catch (error) {
112 if (attempt === maxRetries) throw error;
113
114 // Calculate backoff
115 const delay = backoff === 'exponential'
116 ? Math.pow(2, attempt) * 1000 // 2s, 4s, 8s
117 : 1000 * attempt; // 1s, 2s, 3s
118
119 this.logger.warn(`Retry attempt ${attempt}/${maxRetries}`, {
120 delay,
121 error: error.message
122 });
123
124 await this.sleep(delay);
125 }
126 }
127 }
128
129 // Circuit breaker pattern
130 createCircuitBreaker(options = {}) {
131 let failureCount = 0;
132 let lastFailureTime = null;
133 let state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
134
135 const threshold = options.threshold || 5;
136 const timeout = options.timeout || 60000; // 1 minute
137
138 return {
139 async execute(fn) {
140 // Check if circuit is open
141 if (state === 'OPEN') {
142 const elapsed = Date.now() - lastFailureTime;
143
144 if (elapsed < timeout) {
145 throw new Error('Circuit breaker is OPEN');
146 }
147
148 // Try half-open
149 state = 'HALF_OPEN';
150 }
151
152 try {
153 const result = await fn();
154
155 // Success - reset
156 if (state === 'HALF_OPEN') {
157 state = 'CLOSED';
158 failureCount = 0;
159 }
160
161 return result;
162
163 } catch (error) {
164 failureCount++;
165 lastFailureTime = Date.now();
166
167 if (failureCount >= threshold) {
168 state = 'OPEN';
169 this.logger.error('Circuit breaker opened', {
170 failureCount,
171 threshold
172 });
173 }
174
175 throw error;
176 }
177 },
178
179 getState: () => ({ state, failureCount })
180 };
181 }
182}Handle all possible failure modes gracefully - API errors, timeouts, rate limits, invalid inputs, and unexpected responses.
Error Handling Strategy:
1// Complete error handling system
2
3class AIErrorHandler {
4 constructor() {
5 this.logger = new Logger('error-handler');
6 this.metrics = new MetricsCollector();
7 }
8
9 // Handle all error types
10 async handleError(error, request) {
11 // Classify error
12 const errorType = this.classifyError(error);
13
14 // Log with context
15 this.logger.error('AI request failed', {
16 errorType,
17 message: error.message,
18 request: this.sanitizeRequest(request),
19 stack: error.stack
20 });
21
22 // Record metric
23 this.metrics.recordError({
24 type: errorType,
25 provider: request.provider,
26 model: request.model
27 });
28
29 // Return appropriate response
30 switch (errorType) {
31 case 'RATE_LIMIT':
32 return this.handleRateLimit(error, request);
33
34 case 'TIMEOUT':
35 return this.handleTimeout(error, request);
36
37 case 'INVALID_REQUEST':
38 return this.handleInvalidRequest(error, request);
39
40 case 'API_ERROR':
41 return this.handleAPIError(error, request);
42
43 case 'CONTENT_POLICY':
44 return this.handleContentPolicy(error, request);
45
46 default:
47 return this.handleUnknownError(error, request);
48 }
49 }
50
51 // Classify error type
52 classifyError(error) {
53 if (error.status === 429) return 'RATE_LIMIT';
54 if (error.code === 'ECONNABORTED') return 'TIMEOUT';
55 if (error.status === 400) return 'INVALID_REQUEST';
56 if (error.status === 401 || error.status === 403) return 'AUTH_ERROR';
57 if (error.message?.includes('content_policy')) return 'CONTENT_POLICY';
58 if (error.status >= 500) return 'API_ERROR';
59 return 'UNKNOWN';
60 }
61
62 // Handle rate limiting
63 async handleRateLimit(error, request) {
64 // Extract retry-after header
65 const retryAfter = error.headers?.['retry-after'] || 60;
66
67 this.logger.warn('Rate limited', {
68 retryAfter,
69 userId: request.userId
70 });
71
72 // Queue for later
73 await this.queueForRetry(request, retryAfter);
74
75 return {
76 success: false,
77 error: 'RATE_LIMIT',
78 message: 'Request rate limit exceeded. Your request has been queued.',
79 retryAfter: retryAfter
80 };
81 }
82
83 // Handle timeout
84 async handleTimeout(error, request) {
85 // Try with shorter context or simpler model
86 if (request.retryCount < 2) {
87 return await this.retryWithShorterContext(request);
88 }
89
90 return {
91 success: false,
92 error: 'TIMEOUT',
93 message: 'Request timed out. Please try a shorter query.',
94 suggestion: 'Break your request into smaller parts'
95 };
96 }
97
98 // Handle invalid request
99 handleInvalidRequest(error, request) {
100 return {
101 success: false,
102 error: 'INVALID_REQUEST',
103 message: error.message || 'Invalid request format',
104 details: this.extractErrorDetails(error)
105 };
106 }
107
108 // Handle API errors (500s)
109 async handleAPIError(error, request) {
110 // Notify on-call if persistent
111 if (this.isRepeatedFailure(error)) {
112 await this.alertOncall({
113 severity: 'HIGH',
114 message: 'AI API experiencing repeated failures',
115 error: error.message
116 });
117 }
118
119 // Return fallback response
120 return {
121 success: false,
122 error: 'API_ERROR',
123 message: 'Our AI service is temporarily unavailable. Please try again.',
124 fallbackResponse: await this.getFallbackResponse(request)
125 };
126 }
127
128 // Graceful degradation
129 async getFallbackResponse(request) {
130 // Try cached similar responses
131 const similar = await this.findSimilarCachedResponse(request);
132 if (similar) {
133 return {
134 ...similar,
135 warning: 'This is a cached response due to temporary service issues'
136 };
137 }
138
139 // Return helpful error message
140 return {
141 message: "I'm temporarily unable to process your request. Please try again in a moment."
142 };
143 }
144}
145
146// Error recovery strategies
147class ErrorRecovery {
148 // Automatic retry with degradation
149 async retryWithDegradation(request) {
150 const strategies = [
151 () => this.execute(request), // Original
152 () => this.executeWithShorterContext(request), // Reduce context
153 () => this.executeWithCheaperModel(request), // Cheaper model
154 () => this.executeWithSimplifiedPrompt(request) // Simplified
155 ];
156
157 for (const strategy of strategies) {
158 try {
159 return await strategy();
160 } catch (error) {
161 continue; // Try next strategy
162 }
163 }
164
165 throw new Error('All retry strategies failed');
166 }
167
168 // Queue for background processing
169 async queueForRetry(request, delaySeconds) {
170 await this.queue.add('ai-request', request, {
171 delay: delaySeconds * 1000,
172 attempts: 3,
173 backoff: {
174 type: 'exponential',
175 delay: 2000
176 }
177 });
178 }
179}Monitor AI systems in real-time with comprehensive metrics, logging, and alerting.
Monitoring System:
1// Production monitoring and alerting
2
3class AIMonitoring {
4 constructor() {
5 this.metrics = new PrometheusMetrics();
6 this.logger = new StructuredLogger();
7 this.alertManager = new AlertManager();
8 }
9
10 // Track all AI requests
11 recordRequest(data) {
12 // Performance metrics
13 this.metrics.histogram('ai_request_duration', data.duration, {
14 model: data.model,
15 success: data.success
16 });
17
18 // Cost tracking
19 this.metrics.counter('ai_request_cost', data.cost, {
20 model: data.model
21 });
22
23 // Token usage
24 this.metrics.counter('ai_tokens_used', data.tokens, {
25 type: 'input',
26 model: data.model
27 });
28
29 // Success rate
30 this.metrics.counter('ai_requests_total', 1, {
31 model: data.model,
32 status: data.success ? 'success' : 'error'
33 });
34
35 // Cache hit rate
36 if (data.cached) {
37 this.metrics.counter('ai_cache_hits', 1);
38 }
39
40 // Check thresholds and alert
41 this.checkAnomalies(data);
42 }
43
44 // Anomaly detection and alerting
45 async checkAnomalies(data) {
46 // High latency
47 if (data.duration > 10000) { // 10 seconds
48 await this.alertManager.send({
49 severity: 'WARNING',
50 title: 'High AI latency detected',
51 message: `Request took ${data.duration}ms`,
52 model: data.model
53 });
54 }
55
56 // High cost
57 if (data.cost > 1.0) { // $1 per request
58 await this.alertManager.send({
59 severity: 'WARNING',
60 title: 'High cost request',
61 message: `Request cost $${data.cost}`,
62 model: data.model
63 });
64 }
65
66 // Check error rate
67 const errorRate = await this.getErrorRate('5m');
68 if (errorRate > 0.1) { // 10% error rate
69 await this.alertManager.send({
70 severity: 'CRITICAL',
71 title: 'High AI error rate',
72 message: `Error rate: ${(errorRate * 100).toFixed(1)}%`
73 });
74 }
75
76 // Check budget
77 const dailySpend = await this.getDailySpend();
78 const budget = parseFloat(process.env.DAILY_AI_BUDGET);
79
80 if (dailySpend > budget * 0.9) {
81 await this.alertManager.send({
82 severity: 'HIGH',
83 title: 'AI budget threshold reached',
84 message: `Daily spend: $${dailySpend} (budget: $${budget})`
85 });
86 }
87 }
88
89 // Real-time dashboard data
90 async getDashboardMetrics() {
91 const now = Date.now();
92 const hourAgo = now - 3600000;
93
94 return {
95 requests: {
96 total: await this.metrics.query('ai_requests_total', hourAgo),
97 success: await this.metrics.query('ai_requests_success', hourAgo),
98 failed: await this.metrics.query('ai_requests_failed', hourAgo)
99 },
100 performance: {
101 avgLatency: await this.metrics.query('avg(ai_request_duration)', hourAgo),
102 p95Latency: await this.metrics.query('p95(ai_request_duration)', hourAgo),
103 p99Latency: await this.metrics.query('p99(ai_request_duration)', hourAgo)
104 },
105 cost: {
106 hourly: await this.getHourlyCost(),
107 daily: await this.getDailySpend(),
108 monthly: await this.getMonthlySpend()
109 },
110 cache: {
111 hitRate: await this.getCacheHitRate(),
112 savings: await this.getCacheSavings()
113 }
114 };
115 }
116
117 // Structured logging
118 logRequest(level, message, context) {
119 this.logger.log(level, message, {
120 ...context,
121 timestamp: new Date().toISOString(),
122 service: 'ai-service',
123 environment: process.env.NODE_ENV
124 });
125 }
126}Complete production AI service with error handling, monitoring, fallbacks, and incident response.
1// Production AI Service
2class ProductionAI {
3 constructor() {
4 this.openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
5 this.fallback = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
6 this.cache = new Redis(process.env.REDIS_URL);
7 this.monitor = new AIMonitoring();
8 this.errorHandler = new AIErrorHandler();
9 }
10
11 async query(prompt, options = {}) {
12 const requestId = generateId();
13 const startTime = Date.now();
14
15 try {
16 // Check cache
17 const cached = await this.cache.get(prompt);
18 if (cached) {
19 this.monitor.recordCacheHit();
20 return { ...JSON.parse(cached), cached: true };
21 }
22
23 // Execute with retry and fallback
24 const result = await this.executeWithFallback(prompt, options);
25
26 // Cache result
27 await this.cache.setex(prompt, 3600, JSON.stringify(result));
28
29 // Record metrics
30 this.monitor.recordRequest({
31 requestId,
32 duration: Date.now() - startTime,
33 success: true,
34 cost: result.cost
35 });
36
37 return result;
38 } catch (error) {
39 return this.errorHandler.handleError(error, { prompt, options });
40 }
41 }
42}Production-ready AI service with caching, error handling, monitoring, fallbacks, and cost tracking.
Congratulations! Final lesson: "Building a Complete AI-Powered Application" - put everything together in a production app.
Your progress is saved in this browser, on this device only.