Handling Flaky E2E Tests — Root Causes and Fixes

1// FLAKY: hardcoded sleep — fails when the server is slow
2test('shows confirmation after payment', async ({ page }) => {
3  await page.getByRole('button', { name: 'Pay Now' }).click();
4  await page.waitForTimeout(3000); // NEVER DO THIS
5  await expect(page.getByText('Payment successful')).toBeVisible();
6});
7
8// FIXED: wait for the specific network response
9test('shows confirmation after payment', async ({ page }) => {
10  const responsePromise = page.waitForResponse(
11    (res) => res.url().includes('/api/payments/confirm') && res.status() === 200
12  );
13  await page.getByRole('button', { name: 'Pay Now' }).click();
14  await responsePromise; // waits for the actual response, not a fixed time
15  await expect(page.getByText('Payment successful')).toBeVisible();
16});
17
18// FLAKY: click before element is interactive
19await page.click('.submit-btn'); // may click before event handlers attach
20
21// FIXED: use role-based locator which waits for enabled state
22await page.getByRole('button', { name: 'Submit' }).click();
23// Playwright auto-waits for: visible, enabled, stable, receiving events
24
25// FLAKY: assert before async data loads
26test('shows course count', async ({ page }) => {
27  await page.goto('/courses');
28  await expect(page.getByTestId('course-count')).toHaveText('25'); // may still show '...' loading
29});
30
31// FIXED: wait for network to settle
32test('shows course count', async ({ page }) => {
33  await page.goto('/courses');
34  await page.waitForLoadState('networkidle'); // all network requests complete
35  await expect(page.getByTestId('course-count')).toHaveText('25');
36});

1// BRITTLE: CSS class selector — breaks on any refactor
2await page.click('.btn.btn-primary.mt-4'); // class names change constantly
3await page.click('div:nth-child(3) > button'); // position-based — breaks on reorder
4await page.click('[data-v-3f8a9b12]'); // Vue scoped style hash — changes on build
5
6// RESILIENT: semantic locators that survive refactors
7await page.getByRole('button', { name: 'Enrol Now' }).click();
8await page.getByLabel('Email address').fill('[email protected]');
9await page.getByText('TypeScript Fundamentals').click();
10
11// When semantic locators are insufficient, use stable data-testid attributes:
12// In production code: <button data-testid="checkout-submit-btn">Pay Now</button>
13await page.getByTestId('checkout-submit-btn').click();
14// data-testid is a stable contract you control — CSS classes are not
15
16// BRITTLE: expects exact child count that may change
17await expect(page.locator('.course-grid > *')).toHaveCount(9);
18// This fails whenever a new course is added
19
20// RESILIENT: assert on minimum or specific known content
21await expect(page.getByRole('article')).toHaveCount({ minimum: 1 });
22// OR assert on specific items you control
23await expect(page.getByRole('article').filter({ hasText: 'TypeScript' })).toHaveCount(1);

1// FLAKY: tests share authentication state
2test.describe('User settings', () => {
3  // No explicit setup — depends on a previous test having logged in
4  test('updates profile', async ({ page }) => {
5    await page.goto('/settings'); // may redirect to /login if previous test failed
6    await page.getByLabel('Name').fill('Alice');
7  });
8});
9
10// FIXED: each test or describe block manages its own auth state
11test.describe('User settings', () => {
12  test.use({ storageState: 'e2e/.auth/student.json' }); // explicit auth per describe
13
14  test('updates profile', async ({ page }) => {
15    await page.goto('/settings');
16    await page.getByLabel('Name').fill('Alice');
17  });
18});
19
20// FLAKY: test A creates data, test B depends on that data
21test('creates a draft course', async ({ page }) => {
22  await page.goto('/instructor/courses/new');
23  await page.getByLabel('Title').fill('My Course');
24  await page.getByRole('button', { name: 'Save draft' }).click();
25  // Data left in database
26});
27
28test('draft appears in My Courses', async ({ page }) => {
29  // Assumes previous test ran and left the draft — ORDER DEPENDENT
30  await page.goto('/instructor/courses');
31  await expect(page.getByText('My Course')).toBeVisible();
32});
33
34// FIXED: each test seeds and cleans up its own data
35test('draft appears in My Courses after creation', async ({ page, request }) => {
36  // Setup: create via API
37  const res = await request.post('/api/courses', { data: { title: 'Self-Contained Course', ... } });
38  const { id } = await res.json();
39
40  // Act: verify in browser
41  await page.goto('/instructor/courses');
42  await expect(page.getByText('Self-Contained Course')).toBeVisible();
43
44  // Teardown: always clean up
45  await request.delete(`/api/courses/${id}`, { headers: { Authorization: `Bearer ${token}` } });
46});

1// FLAKY: makes real Stripe API call — fails when Stripe is slow
2test('payment form works', async ({ page }) => {
3  await page.goto('/checkout');
4  await page.getByLabel('Card number').fill('4242424242424242');
5  await page.getByLabel('Expiry').fill('12/28');
6  await page.getByLabel('CVC').fill('123');
7  await page.getByRole('button', { name: 'Pay' }).click();
8  // Real Stripe API call — may timeout on slow CI
9  await expect(page).toHaveURL('/checkout/success', { timeout: 30_000 });
10});
11
12// FIXED: mock external payment API
13test('payment form submits and shows success', async ({ page }) => {
14  await page.route('**/api/stripe/**', (route) =>
15    route.fulfill({ body: JSON.stringify({ status: 'succeeded', chargeId: 'ch_test' }) })
16  );
17  await page.route('https://api.stripe.com/**', (route) =>
18    route.fulfill({ body: JSON.stringify({ id: 'pi_test', status: 'succeeded' }) })
19  );
20
21  await page.goto('/checkout');
22  await page.getByLabel('Card number').fill('4242424242424242');
23  await page.getByLabel('Expiry').fill('12/28');
24  await page.getByLabel('CVC').fill('123');
25  await page.getByRole('button', { name: 'Pay' }).click();
26  await expect(page).toHaveURL('/checkout/success');
27});
28
29// For tests that MUST use real external services:
30// Set a longer timeout and mark as @slow
31test('@slow real Stripe integration test', async ({ page }) => {
32  test.setTimeout(60_000); // 60 second timeout for this test only
33  // ... real Stripe test ...
34});

1# Run a test file multiple times to expose flakiness:
2npx playwright test e2e/checkout.spec.ts --repeat-each=5
3
4# Run tests in random order to detect ordering dependencies:
5npx playwright test --repeat-each=3 --workers=1
6# (workers=1 prevents parallel execution masking ordering issues)
7
8# Enable full trace for a specific test:
9npx playwright test e2e/flaky.spec.ts --trace=on
10# Then open: npx playwright show-report
11# Look for: timing between actions, network response times, DOM state changes

1// test.fixme — marks a test as broken and skips it with documentation
2test.fixme('checkout fails on mobile Safari', async ({ page }) => {
3  // https://github.com/org/repo/issues/456 — WebKit payment API issue
4  // DO NOT just comment out or skip without a reason
5});
6// test.fixme shows up in the HTML report with count of broken tests
7// it is a formal acknowledgement that you know a test is broken
8
9// test retries — use to handle GENUINE infrastructure flakiness, not code bugs
10// playwright.config.ts:
11retries: process.env['CI'] ? 2 : 0,
12// 2 retries in CI catches transient failures (container resource pressure)
13// 0 retries locally so you see failures immediately without false hope
14
15// WRONG: using retries to hide a real flaky test
16test('this test sometimes fails for unknown reasons', async ({ page }) => {
17  test.info().retry; // checking retry count inside a test is a code smell
18  // If a test needs special logic on retry, it is masking a real problem
19});
20
21// RIGHT: investigate the failure with trace viewer, fix the root cause
22// Only use retries for infrastructure-level issues (network blips, CI resource contention)