Master sentiment analysis techniques from lexicon-based approaches to machine learning models for analyzing opinions, emotions, and attitudes in text.
Sentiment analysis (opinion mining) automatically determines the emotional tone behind text - whether it's positive, negative, or neutral. This technology powers product review analysis, social media monitoring, brand reputation tracking, and customer feedback systems.
There are three main approaches: lexicon-based methods that use predefined sentiment dictionaries, machine learning models trained on labeled data, and deep learning approaches using neural networks. Each has trade-offs between simplicity, accuracy, and domain adaptability.
In this lesson, you'll implement sentiment analyzers using multiple approaches and learn when to use each.
Uses predefined word lists with sentiment scores.
1class LexiconSentimentAnalyzer:
2 def __init__(self):
3 # Simplified sentiment lexicon
4 self.lexicon = {
5 # Positive words (score > 0)
6 'good': 2, 'great': 3, 'excellent': 4, 'amazing': 4,
7 'wonderful': 4, 'fantastic': 4, 'love': 3, 'best': 3,
8 'happy': 2, 'perfect': 4, 'awesome': 3, 'beautiful': 3,
9
10 # Negative words (score < 0)
11 'bad': -2, 'terrible': -4, 'awful': -4, 'horrible': -4,
12 'worst': -4, 'hate': -3, 'poor': -2, 'disappointing': -3,
13 'sad': -2, 'angry': -2, 'ugly': -2, 'boring': -2,
14
15 # Intensifiers (multiply next word)
16 'very': 1.5, 'extremely': 2.0, 'really': 1.3,
17
18 # Negators (flip sentiment)
19 'not': -1, "n't": -1, 'no': -1, 'never': -1,
20 }
21
22 self.negators = {'not', "n't", 'no', 'never', 'none', 'neither'}
23 self.intensifiers = {'very', 'extremely', 'really', 'so', 'too'}
24
25 def analyze(self, text):
26 words = text.lower().split()
27 score = 0
28 modifier = 1
29
30 for i, word in enumerate(words):
31 # Check for negation
32 if word in self.negators:
33 modifier *= -1
34 continue
35
36 # Check for intensifier
37 if word in self.intensifiers:
38 modifier *= self.lexicon.get(word, 1.5)
39 continue
40
41 # Get word sentiment
42 if word in self.lexicon:
43 word_score = self.lexicon[word]
44 if word_score > 0 or word_score < 0: # Skip modifiers
45 score += word_score * modifier
46 modifier = 1 # Reset modifier
47
48 return {'score': score, 'sentiment': self._classify(score)}
49
50 def _classify(self, score):
51 if score > 1: return 'positive'
52 elif score < -1: return 'negative'
53 return 'neutral'Negation and context significantly affect sentiment interpretation.
1def handle_negation(tokens, sentiment_scores):
2 """Handle negation by flipping sentiment in window"""
3 negation_window = 3 # Words affected after negator
4 negators = {'not', "n't", 'no', 'never', 'none', 'neither', 'nor'}
5
6 scores = []
7 negate = 0 # Countdown for negation window
8
9 for token in tokens:
10 if token.lower() in negators:
11 negate = negation_window
12 scores.append(0) # Negator itself is neutral
13 elif negate > 0:
14 # Flip sentiment
15 original_score = sentiment_scores.get(token.lower(), 0)
16 scores.append(-original_score)
17 negate -= 1
18 else:
19 scores.append(sentiment_scores.get(token.lower(), 0))
20
21 return scores
22
23# Example
24text = "This is not good at all"
25tokens = text.split()
26lexicon = {'good': 3, 'bad': -3}
27
28# Without negation handling
29naive_score = sum(lexicon.get(t.lower(), 0) for t in tokens)
30print(f"Naive: {naive_score}") # 3 (wrong!)
31
32# With negation handling
33scores = handle_negation(tokens, lexicon)
34print(f"With negation: {sum(scores)}") # -3 (correct!)Training classifiers on labeled sentiment data.
1from sklearn.feature_extraction.text import TfidfVectorizer
2from sklearn.linear_model import LogisticRegression
3from sklearn.pipeline import Pipeline
4
5# Sentiment classification pipeline
6sentiment_pipeline = Pipeline([
7 ('tfidf', TfidfVectorizer(
8 max_features=10000,
9 ngram_range=(1, 2), # Unigrams and bigrams
10 min_df=2
11 )),
12 ('classifier', LogisticRegression(
13 C=1.0,
14 max_iter=1000,
15 class_weight='balanced'
16 ))
17])
18
19# Train
20sentiment_pipeline.fit(X_train, y_train)
21
22# Predict with probabilities
23predictions = sentiment_pipeline.predict(X_test)
24probabilities = sentiment_pipeline.predict_proba(X_test)
25
26# Feature importance for interpretability
27feature_names = sentiment_pipeline['tfidf'].get_feature_names_out()
28coefficients = sentiment_pipeline['classifier'].coef_[0]
29
30# Top positive/negative features
31positive_features = [feature_names[i] for i in coefficients.argsort()[-10:]]
32negative_features = [feature_names[i] for i in coefficients.argsort()[:10]]Analyzing sentiment toward specific aspects of products/services.
1def aspect_sentiment(text, aspects, analyzer):
2 """
3 Extract sentiment for specific aspects mentioned in text.
4
5 Example: "The food was great but the service was terrible"
6 - food: positive
7 - service: negative
8 """
9 results = {}
10 sentences = text.split('.')
11
12 for aspect in aspects:
13 aspect_sentiment_scores = []
14
15 for sentence in sentences:
16 if aspect.lower() in sentence.lower():
17 # Analyze sentiment of sentence containing aspect
18 result = analyzer.analyze(sentence)
19 aspect_sentiment_scores.append(result['score'])
20
21 if aspect_sentiment_scores:
22 avg_score = sum(aspect_sentiment_scores) / len(aspect_sentiment_scores)
23 results[aspect] = {
24 'score': avg_score,
25 'sentiment': 'positive' if avg_score > 0 else 'negative' if avg_score < 0 else 'neutral',
26 'mentions': len(aspect_sentiment_scores)
27 }
28
29 return results
30
31# Example
32review = "The food was delicious and fresh. However, the service was slow and rude. The ambiance was nice."
33aspects = ['food', 'service', 'ambiance']
34# Would return different sentiments for each aspectBuild a hybrid sentiment analyzer combining lexicon and ML approaches.
1import re
2from collections import Counter
3
4class SentimentAnalyzer:
5 """Hybrid sentiment analysis system"""
6
7 def __init__(self):
8 # Sentiment lexicon
9 self.positive_words = {
10 'good': 2, 'great': 3, 'excellent': 4, 'amazing': 4,
11 'wonderful': 4, 'love': 3, 'best': 3, 'happy': 2,
12 'perfect': 4, 'awesome': 3, 'fantastic': 4, 'beautiful': 3,
13 'recommend': 2, 'satisfied': 2, 'impressed': 3
14 }
15
16 self.negative_words = {
17 'bad': -2, 'terrible': -4, 'awful': -4, 'horrible': -4,
18 'worst': -4, 'hate': -3, 'poor': -2, 'disappointing': -3,
19 'sad': -2, 'angry': -2, 'boring': -2, 'waste': -3,
20 'broken': -3, 'useless': -3, 'avoid': -2
21 }
22
23 self.intensifiers = {
24 'very': 1.5, 'extremely': 2.0, 'really': 1.5, 'so': 1.3,
25 'absolutely': 2.0, 'completely': 1.8, 'totally': 1.8
26 }
27
28 self.negators = {'not', "n't", 'no', 'never', 'none', 'neither', "don't", "doesn't", "didn't", "won't", "wouldn't", "couldn't", "shouldn't"}
29
30 def preprocess(self, text):
31 """Clean and tokenize text"""
32 text = text.lower()
33 text = re.sub(r'[^a-z\s\'']', ' ', text)
34 tokens = text.split()
35 return tokens
36
37 def lexicon_score(self, tokens):
38 """Calculate sentiment score using lexicon"""
39 score = 0
40 modifier = 1
41 negation_window = 0
42
43 for i, token in enumerate(tokens):
44 # Handle negation
45 if token in self.negators:
46 negation_window = 3
47 continue
48
49 # Handle intensifiers
50 if token in self.intensifiers:
51 modifier *= self.intensifiers[token]
52 continue
53
54 # Calculate word score
55 if token in self.positive_words:
56 word_score = self.positive_words[token]
57 elif token in self.negative_words:
58 word_score = self.negative_words[token]
59 else:
60 word_score = 0
61
62 # Apply negation
63 if negation_window > 0 and word_score != 0:
64 word_score = -word_score
65 negation_window -= 1
66
67 # Apply modifier
68 score += word_score * modifier
69 modifier = 1 # Reset
70
71 return score
72
73 def analyze(self, text):
74 """Full sentiment analysis"""
75 tokens = self.preprocess(text)
76 score = self.lexicon_score(tokens)
77
78 # Classify
79 if score >= 2:
80 sentiment = 'positive'
81 confidence = min(abs(score) / 10, 1.0)
82 elif score <= -2:
83 sentiment = 'negative'
84 confidence = min(abs(score) / 10, 1.0)
85 else:
86 sentiment = 'neutral'
87 confidence = 0.5
88
89 # Extract key sentiment words
90 positive_found = [t for t in tokens if t in self.positive_words]
91 negative_found = [t for t in tokens if t in self.negative_words]
92
93 return {
94 'text': text,
95 'sentiment': sentiment,
96 'score': score,
97 'confidence': confidence,
98 'positive_words': positive_found,
99 'negative_words': negative_found
100 }
101
102 def analyze_batch(self, texts):
103 """Analyze multiple texts"""
104 return [self.analyze(text) for text in texts]
105
106# Test the analyzer
107analyzer = SentimentAnalyzer()
108
109reviews = [
110 "This product is absolutely amazing! Best purchase ever.",
111 "Terrible quality, completely disappointed. Total waste of money.",
112 "The product is okay, nothing special but works fine.",
113 "I really love this! Exceeded all my expectations.",
114 "Not good at all. Very disappointing experience.",
115 "Great value for the price. Would definitely recommend.",
116 "The worst product I've ever bought. Avoid at all costs.",
117 "It's not bad, but could be better.",
118]
119
120print("SENTIMENT ANALYSIS SYSTEM")
121print("=" * 70)
122
123# Analyze each review
124results = analyzer.analyze_batch(reviews)
125
126for result in results:
127 print(f"\nText: {result['text'][:50]}...")
128 print(f" Sentiment: {result['sentiment'].upper()} (score: {result['score']:.1f}, conf: {result['confidence']:.2f})")
129 if result['positive_words']:
130 print(f" Positive words: {result['positive_words']}")
131 if result['negative_words']:
132 print(f" Negative words: {result['negative_words']}")
133
134# Summary statistics
135print("\n" + "=" * 70)
136print("SUMMARY:")
137sentiment_counts = Counter(r['sentiment'] for r in results)
138for sentiment, count in sentiment_counts.items():
139 print(f" {sentiment}: {count} ({count/len(results)*100:.0f}%)")
140
141avg_score = sum(r['score'] for r in results) / len(results)
142print(f"\nAverage sentiment score: {avg_score:.2f}")This demonstrates a complete sentiment analysis system with lexicon-based scoring, negation handling, intensifiers, and confidence estimation.
In our next lesson, "Topic Modeling," you'll learn to discover hidden themes in document collections using LDA and other unsupervised techniques.
Your progress is saved in this browser, on this device only.