Merge pull request #29041 from storybookjs/jeppe/fix-e2e-flake

Build: Improve E2E tests with ESLint rules
storybookjs · Sep 4, 2024 · ab66906 · ab66906
2 parents 5e0d8b0 + bb3301c
commit ab66906
Show file tree

Hide file tree

Showing 20 changed files with 266 additions and 259 deletions.
diff --git a/code/.eslintrc.js b/code/.eslintrc.js
@@ -177,5 +177,28 @@ module.exports = {
         'local-rules/no-duplicated-error-codes': 'error',
       },
     },
+    {
+      files: ['./e2e-tests/*.ts'],
+      extends: ['plugin:playwright/recommended'],
+      rules: {
+        'playwright/no-skipped-test': [
+          'warn',
+          {
+            allowConditional: true,
+          },
+        ],
+        'playwright/no-raw-locators': 'off', // TODO: enable this, requires the UI to actually be accessible
+        'playwright/prefer-comparison-matcher': 'error',
+        'playwright/prefer-equality-matcher': 'error',
+        'playwright/prefer-hooks-on-top': 'error',
+        'playwright/prefer-strict-equal': 'error',
+        'playwright/prefer-to-be': 'error',
+        'playwright/prefer-to-contain': 'error',
+        'playwright/prefer-to-have-count': 'error',
+        'playwright/prefer-to-have-length': 'error',
+        'playwright/require-to-throw-message': 'error',
+        'playwright/require-top-level-describe': 'error',
+      },
+    },
   ],
 };
diff --git a/code/addons/test/package.json b/code/addons/test/package.json
@@ -8,14 +8,14 @@
     "vitest",
     "testing"
   ],
-  "homepage": "https://github.com/storybookjs/storybook/tree/next/code/addons/vitest",
+  "homepage": "https://github.com/storybookjs/storybook/tree/next/code/addons/test",
   "bugs": {
     "url": "https://github.com/storybookjs/storybook/issues"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/storybookjs/storybook.git",
-    "directory": "code/addons/vitest"
+    "directory": "code/addons/test"
   },
   "funding": {
     "type": "opencollective",

diff --git a/code/e2e-tests/addon-actions.spec.ts b/code/e2e-tests/addon-actions.spec.ts
@@ -18,11 +18,11 @@ test.describe('addon-actions', () => {
 
     await sbPage.navigateToStory('example/button', 'primary');
     const root = sbPage.previewRoot();
-    const button = root.locator('button', { hasText: 'Button' });
+    const button = root.getByRole('button', { name: 'Button' });
     await button.click();
 
     await sbPage.viewAddonPanel('Actions');
-    const logItem = await page.locator('#storybook-panel-root #panel-tab-content', {
+    const logItem = page.locator('#storybook-panel-root #panel-tab-content', {
       hasText: 'click',
     });
     await expect(logItem).toBeVisible();
@@ -40,11 +40,11 @@ test.describe('addon-actions', () => {
     await sbPage.navigateToStory('addons/actions/spies', 'show-spy-on-in-actions');
 
     const root = sbPage.previewRoot();
-    const button = root.locator('button', { hasText: 'Button' });
+    const button = root.getByRole('button', { name: 'Button' });
     await button.click();
 
     await sbPage.viewAddonPanel('Actions');
-    const logItem = await page.locator('#storybook-panel-root #panel-tab-content', {
+    const logItem = page.locator('#storybook-panel-root #panel-tab-content', {
       hasText: 'console.log',
     });
     await expect(logItem).toBeVisible();

diff --git a/code/e2e-tests/addon-controls.spec.ts b/code/e2e-tests/addon-controls.spec.ts
@@ -21,9 +21,7 @@ test.describe('addon-controls', () => {
     await expect(sbPage.previewRoot().locator('button')).toContainText('Hello world');
 
     // Args in URL
-    await page.waitForTimeout(300);
-    const url = await page.url();
-    await expect(url).toContain('args=label:Hello+world');
+    await page.waitForURL((url) => url.search.includes('args=label:Hello+world'));
 
     // Boolean toggle: Primary/secondary
     await expect(sbPage.previewRoot().locator('button')).toHaveCSS(
@@ -72,8 +70,8 @@ test.describe('addon-controls', () => {
     await expect(sbPage.previewRoot().locator('button')).toContainText('Hello world');
 
     await sbPage.viewAddonPanel('Controls');
-    const label = await sbPage.panelContent().locator('textarea[name=label]').inputValue();
-    await expect(label).toEqual('Hello world');
+    const label = sbPage.panelContent().locator('textarea[name=label]');
+    await expect(label).toHaveValue('Hello world');
   });
 
   test('should set select option when value contains double spaces', async ({ page }) => {

diff --git a/code/e2e-tests/addon-docs.spec.ts b/code/e2e-tests/addon-docs.spec.ts
@@ -1,3 +1,6 @@
+/* eslint-disable playwright/no-conditional-expect */
+
+/* eslint-disable playwright/no-conditional-in-test */
 import { expect, test } from '@playwright/test';
 import process from 'process';
 import { dedent } from 'ts-dedent';
@@ -94,14 +97,14 @@ test.describe('addon-docs', () => {
 
     const toggleCount = await toggles.count();
     for (let i = 0; i < toggleCount; i += 1) {
-      const toggle = await toggles.nth(i);
-      await toggle.click({ force: true });
+      const toggle = toggles.nth(i);
+      await toggle.click();
     }
 
     const codes = root.locator('pre.prismjs');
     const codeCount = await codes.count();
     for (let i = 0; i < codeCount; i += 1) {
-      const code = await codes.nth(i);
+      const code = codes.nth(i);
       const text = await code.innerText();
       await expect(text).not.toMatch(/^\(args\) => /);
     }
@@ -132,13 +135,13 @@ test.describe('addon-docs', () => {
     const toggles = root.locator('.docblock-code-toggle');
 
     // Open up the first and second code toggle (i.e the "Basic" story outside and inside the Stories block)
-    await (await toggles.nth(0)).click({ force: true });
-    await (await toggles.nth(1)).click({ force: true });
+    await toggles.nth(0).click();
+    await toggles.nth(1).click();
 
     // Check they both say "Basic"
     const codes = root.locator('pre.prismjs');
-    const primaryCode = await codes.nth(0);
-    const storiesCode = await codes.nth(1);
+    const primaryCode = codes.nth(0);
+    const storiesCode = codes.nth(1);
     await expect(primaryCode).toContainText('Basic');
     await expect(storiesCode).toContainText('Basic');
 
@@ -179,7 +182,7 @@ test.describe('addon-docs', () => {
     const root = sbPage.previewRoot();
     const stories = root.locator('.sb-story button');
 
-    await expect(await stories.count()).toBe(3);
+    await expect(stories).toHaveCount(3);
     await expect(stories.first()).toHaveText('Basic');
     await expect(stories.nth(1)).toHaveText('Basic');
     await expect(stories.last()).toHaveText('Another');
@@ -210,12 +213,12 @@ test.describe('addon-docs', () => {
     }
 
     // Arrange - Get the actual versions
-    const mdxReactVersion = await root.getByTestId('mdx-react');
-    const mdxReactDomVersion = await root.getByTestId('mdx-react-dom');
-    const mdxReactDomServerVersion = await root.getByTestId('mdx-react-dom-server');
-    const componentReactVersion = await root.getByTestId('component-react');
-    const componentReactDomVersion = await root.getByTestId('component-react-dom');
-    const componentReactDomServerVersion = await root.getByTestId('component-react-dom-server');
+    const mdxReactVersion = root.getByTestId('mdx-react');
+    const mdxReactDomVersion = root.getByTestId('mdx-react-dom');
+    const mdxReactDomServerVersion = root.getByTestId('mdx-react-dom-server');
+    const componentReactVersion = root.getByTestId('component-react');
+    const componentReactDomVersion = root.getByTestId('component-react-dom');
+    const componentReactDomServerVersion = root.getByTestId('component-react-dom-server');
 
     // Assert - The versions are in the expected range
     await expect(mdxReactVersion).toHaveText(expectedReactVersionRange);
@@ -232,9 +235,9 @@ test.describe('addon-docs', () => {
     await sbPage.navigateToStory('addons/docs/docs2/resolvedreact', 'docs');
 
     // Arrange - Get the actual versions
-    const autodocsReactVersion = await root.getByTestId('react');
-    const autodocsReactDomVersion = await root.getByTestId('react-dom');
-    const autodocsReactDomServerVersion = await root.getByTestId('react-dom-server');
+    const autodocsReactVersion = root.getByTestId('react');
+    const autodocsReactDomVersion = root.getByTestId('react-dom');
+    const autodocsReactDomServerVersion = root.getByTestId('react-dom-server');
 
     // Assert - The versions are in the expected range
     await expect(autodocsReactVersion).toHaveText(expectedReactVersionRange);
@@ -247,9 +250,9 @@ test.describe('addon-docs', () => {
     await sbPage.navigateToStory('addons/docs/docs2/resolvedreact', 'story');
 
     // Arrange - Get the actual versions
-    const storyReactVersion = await root.getByTestId('react');
-    const storyReactDomVersion = await root.getByTestId('react-dom');
-    const storyReactDomServerVersion = await root.getByTestId('react-dom-server');
+    const storyReactVersion = root.getByTestId('react');
+    const storyReactDomVersion = root.getByTestId('react-dom');
+    const storyReactDomServerVersion = root.getByTestId('react-dom-server');
 
     // Assert - The versions are in the expected range
     await expect(storyReactVersion).toHaveText(expectedReactVersionRange);
@@ -265,12 +268,14 @@ test.describe('addon-docs', () => {
     const root = sbPage.previewRoot();
 
     const storyHeadings = root.locator('.sb-anchor > h3');
-    await expect(await storyHeadings.count()).toBe(6);
-    await expect(storyHeadings.nth(0)).toHaveText('Default A');
-    await expect(storyHeadings.nth(1)).toHaveText('Span Content');
-    await expect(storyHeadings.nth(2)).toHaveText('Code Content');
-    await expect(storyHeadings.nth(3)).toHaveText('Default B');
-    await expect(storyHeadings.nth(4)).toHaveText('H 1 Content');
-    await expect(storyHeadings.nth(5)).toHaveText('H 2 Content');
+    await expect(storyHeadings).toHaveCount(6);
+    await expect(storyHeadings).toHaveText([
+      'Default A',
+      'Span Content',
+      'Code Content',
+      'Default B',
+      'H 1 Content',
+      'H 2 Content',
+    ]);
   });
 });
diff --git a/code/e2e-tests/addon-interactions.spec.ts b/code/e2e-tests/addon-interactions.spec.ts
@@ -24,10 +24,10 @@ test.describe('addon-interactions', () => {
     await sbPage.navigateToStory('example/page', 'logged-in');
     await sbPage.viewAddonPanel('Interactions');
 
-    const welcome = await sbPage.previewRoot().locator('.welcome');
+    const welcome = sbPage.previewRoot().locator('.welcome');
     await expect(welcome).toContainText('Welcome, Jane Doe!', { timeout: 50000 });
 
-    const interactionsTab = await page.locator('#tabbutton-storybook-interactions-panel');
+    const interactionsTab = page.locator('#tabbutton-storybook-interactions-panel');
     await expect(interactionsTab).toContainText(/(\d)/);
     await expect(interactionsTab).toBeVisible();
 
@@ -36,7 +36,7 @@ test.describe('addon-interactions', () => {
     await expect(panel).toContainText(/userEvent.click/);
     await expect(panel).toBeVisible();
 
-    const done = await panel.locator('[data-testid=icon-done]').nth(0);
+    const done = panel.locator('[data-testid=icon-done]').nth(0);
     await expect(done).toBeVisible();
   });
 
@@ -57,35 +57,35 @@ test.describe('addon-interactions', () => {
     await sbPage.viewAddonPanel('Interactions');
 
     // Test initial state - Interactions have run, count is correct and values are as expected
-    const formInput = await sbPage.previewRoot().locator('#interaction-test-form input');
+    const formInput = sbPage.previewRoot().locator('#interaction-test-form input');
     await expect(formInput).toHaveValue('final value', { timeout: 50000 });
 
-    const interactionsTab = await page.locator('#tabbutton-storybook-interactions-panel');
+    const interactionsTab = page.locator('#tabbutton-storybook-interactions-panel');
     await expect(interactionsTab.getByText('3')).toBeVisible();
     await expect(interactionsTab).toBeVisible();
     await expect(interactionsTab).toBeVisible();
 
     const panel = sbPage.panelContent();
-    const runStatusBadge = await panel.locator('[aria-label="Status of the test run"]');
+    const runStatusBadge = panel.locator('[aria-label="Status of the test run"]');
     await expect(runStatusBadge).toContainText(/Pass/);
     await expect(panel).toContainText(/"initial value"/);
     await expect(panel).toContainText(/clear/);
     await expect(panel).toContainText(/"final value"/);
     await expect(panel).toBeVisible();
 
     // Test interactions debugger - Stepping through works, count is correct and values are as expected
-    const interactionsRow = await panel.locator('[aria-label="Interaction step"]');
+    const interactionsRow = panel.locator('[aria-label="Interaction step"]');
 
     await interactionsRow.first().isVisible();
 
-    await expect(await interactionsRow.count()).toEqual(3);
+    await expect(interactionsRow).toHaveCount(3);
     const firstInteraction = interactionsRow.first();
     await firstInteraction.click();
 
     await expect(runStatusBadge).toContainText(/Runs/);
     await expect(formInput).toHaveValue('initial value');
 
-    const goForwardBtn = await panel.locator('[aria-label="Go forward"]');
+    const goForwardBtn = panel.locator('[aria-label="Go forward"]');
     await goForwardBtn.click();
     await expect(formInput).toHaveValue('');
     await goForwardBtn.click();
@@ -94,7 +94,7 @@ test.describe('addon-interactions', () => {
     await expect(runStatusBadge).toContainText(/Pass/);
 
     // Test rerun state (from addon panel) - Interactions have rerun, count is correct and values are as expected
-    const rerunInteractionButton = await panel.locator('[aria-label="Rerun"]');
+    const rerunInteractionButton = panel.locator('[aria-label="Rerun"]');
     await rerunInteractionButton.click();
 
     await expect(formInput).toHaveValue('final value');
@@ -107,7 +107,7 @@ test.describe('addon-interactions', () => {
     await expect(interactionsTab.getByText('3')).toBeVisible();
 
     // Test remount state (from toolbar) - Interactions have rerun, count is correct and values are as expected
-    const remountComponentButton = await page.locator('[title="Remount component"]');
+    const remountComponentButton = page.locator('[title="Remount component"]');
     await remountComponentButton.click();
 
     await interactionsRow.first().isVisible();
@@ -132,7 +132,7 @@ test.describe('addon-interactions', () => {
     await sbPage.deepLinkToStory(storybookUrl, 'addons/interactions/unhandled-errors', 'default');
     await sbPage.viewAddonPanel('Interactions');
 
-    const button = await sbPage.previewRoot().locator('button');
+    const button = sbPage.previewRoot().locator('button');
     await expect(button).toContainText('Button', { timeout: 50000 });
 
     const panel = sbPage.panelContent();

diff --git a/code/e2e-tests/addon-toolbars.spec.ts b/code/e2e-tests/addon-toolbars.spec.ts
@@ -30,8 +30,8 @@ test.describe('addon-toolbars', () => {
     // Click on viewport button and select spanish
     await sbPage.navigateToStory('addons/toolbars/globals', 'override-locale');
     await expect(sbPage.previewRoot()).toContainText('안녕하세요');
-    const button = await sbPage.page.locator('[title="Internationalization locale"]');
+    const button = sbPage.page.locator('[title="Internationalization locale"]');
 
-    await expect(await button.getAttribute('disabled')).toBe('');
+    await expect(button).toHaveAttribute('disabled', '');
   });
 });
diff --git a/code/e2e-tests/composition.spec.ts b/code/e2e-tests/composition.spec.ts
@@ -3,48 +3,42 @@ import { expect, test } from '@playwright/test';
 import { SbPage } from './util';
 
 const storybookUrl = process.env.STORYBOOK_URL || 'http://localhost:6006';
-
-// This is a slow test, and (presumably) framework independent, so only run it on one sandbox
-const skipTest = process.env.STORYBOOK_TEMPLATE_NAME !== 'react-vite/default-ts';
+const templateName = process.env.STORYBOOK_TEMPLATE_NAME || '';
 
 test.describe('composition', () => {
+  test.skip(
+    templateName !== 'react-vite/default-ts',
+    'Slow, framework independent test, so only run it on in react-vite/default-ts'
+  );
+
   test.beforeEach(async ({ page }) => {
-    if (skipTest) {
-      return;
-    }
     await page.goto(storybookUrl);
     await new SbPage(page).waitUntilLoaded();
   });
 
   test('should correctly filter composed stories', async ({ page }) => {
-    if (skipTest) {
-      return;
-    }
-
-    // Expect that composed Storybooks are visible
-
     // Expect that composed Storybooks are visible
-    await expect(await page.getByTitle('Storybook 8.0.0')).toBeVisible();
-    await expect(await page.getByTitle('Storybook 7.6.18')).toBeVisible();
+    await expect(page.getByTitle('Storybook 8.0.0')).toBeVisible();
+    await expect(page.getByTitle('Storybook 7.6.18')).toBeVisible();
 
     // Expect composed stories to be available in the sidebar
     await page.locator('[id="storybook\\@8\\.0\\.0_components-badge"]').click();
     await expect(
-      await page.locator('[id="storybook\\@8\\.0\\.0_components-badge--default"]')
+      page.locator('[id="storybook\\@8\\.0\\.0_components-badge--default"]')
     ).toBeVisible();
 
     await page.locator('[id="storybook\\@7\\.6\\.18_components-badge"]').click();
     await expect(
-      await page.locator('[id="storybook\\@7\\.6\\.18_components-badge--default"]')
+      page.locator('[id="storybook\\@7\\.6\\.18_components-badge--default"]')
     ).toBeVisible();
 
     // Expect composed stories `to be available in the search
     await page.getByPlaceholder('Find components').fill('Button');
     await expect(
-      await page.getByRole('option', { name: 'Button Storybook 8.0.0 / @blocks / examples' })
+      page.getByRole('option', { name: 'Button Storybook 8.0.0 / @blocks / examples' })
     ).toBeVisible();
     await expect(
-      await page.getByRole('option', { name: 'Button Storybook 7.6.18 / @blocks / examples' })
+      page.getByRole('option', { name: 'Button Storybook 7.6.18 / @blocks / examples' })
     ).toBeVisible();
   });
 });