Improve e2e test docs and error handling (#8214)

* Improve e2e test docs and error handling

* add more detail

* fix

* downgrade to 1.42.1

Upgrade once this issue fix is released: microsoft/playwright#30312

end-to-end-tests/README.md

@@ -11,8 +11,10 @@ One-time setup:
 
 - Set up your .env file:
   - Copy `.env.example` to `.env.development`.
-  - Fill in the required values for the test user password `E2E_TEST_USER_PASSWORD_UNAFFILIATED` and
-    uncomment `REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST=1`
+  - Fill in the required values:
+    - The test user password `E2E_TEST_USER_PASSWORD_UNAFFILIATED`
+    - Uncomment `REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST=1`
+    - Uncomment `SHADOW_DOM=open`
   - `MV` will determine the manifest version for the both the extension and the tests.
 - Install browsers: Execute `npx playwright install chromium chrome msedge`.
 
@@ -23,8 +25,11 @@ One-time setup:
 - To run tests in interactive UI mode, use `npm run test:e2e -- --ui`. This view shows you the entire test suite and
   allows you to run individual tests in a specific browser.
   - If this is the first time you've run the tests in interactive mode, the tests may be filtered. Expand the collapsed section next to the Filter searchbox to see the filterable "projects"
+  - For faster local development, you can filter out the setup projects to skip rerunning the authentication setup by unticking the `edgeSetup` and `chromeSetup` projects.
 - You can also run specific test files in the CLI by providing a path matcher to the
   command: `npm run test:e2e -- smoke` (runs all tests with "smoke" in the path).
+  - You can skip the auth setup by including the `--no-deps` flag.
+  - See more cli arguments here: https://playwright.dev/docs/test-cli
 - You can also run tests within the Intellij IDE by clicking on the play button next to the test definition. (
   until [this Jetbrains issue](https://youtrack.jetbrains.com/issue/AQUA-711/Provide-a-run-configuration-for-Playwright-tests-in-specs-with-fixture-imports-only)
   is resolved, you must include an unused playwright import as shown
@@ -93,7 +98,8 @@ Playwright's built-in `test` object with extension-specific features.
 ### Playwright Configuration
 
 Configure test execution via `.playwright.config.ts`, including timeout and retry options. The setup
-project `./auth.setup.ts` handles user authentication and saves credentials in `./.auth/user.json`.
+projects run `./auth.setup.ts` which handles user authentication and saves the path to the logged in and linked profile
+paths in `.auth`.
 
 ### GitHub CI Integration
 

end-to-end-tests/fixtures/extensionBase.ts

@@ -36,12 +36,31 @@ export const test = base.extend<{
 }>({
   chromiumChannel: ["chrome", { option: true }],
   async context({ chromiumChannel }, use) {
-    const authSetupProfileDirectory = await fs.readFile(
-      getAuthProfilePathFile(chromiumChannel),
-      "utf8",
-    );
+    let authSetupProfileDirectory: string;
+
+    try {
+      authSetupProfileDirectory = await fs.readFile(
+        getAuthProfilePathFile(chromiumChannel),
+        "utf8",
+      );
+    } catch (error) {
+      if (
+        error instanceof Error &&
+        "code" in error &&
+        error.code === "ENOENT"
+      ) {
+        console.log(
+          "No auth setup profile found. Make sure that the `auth.setup` project has been run first to create the " +
+            "profile. (If using UI mode, make sure that the chromeSetup and/or the edgeSetup projects are not filtered out)",
+        );
+      }
+
+      throw error;
+    }
+
     const temporaryProfileDirectory = await fs.mkdtemp(
-      path.join(path.dirname(authSetupProfileDirectory), "e2e-test-"),
+      // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion,@typescript-eslint/no-non-null-assertion -- checked above
+      path.join(path.dirname(authSetupProfileDirectory!), "e2e-test-"),
     );
     // Copy the auth setup profile to a new temp directory to avoid modifying the original auth profile
     await fs.cp(authSetupProfileDirectory, temporaryProfileDirectory, {

package.json

@@ -180,7 +180,7 @@
   "devDependencies": {
     "@axe-core/playwright": "^4.9.0",
     "@fortawesome/fontawesome-common-types": "^0.2.36",
-    "@playwright/test": "^1.43.0",
+    "@playwright/test": "^1.42.1",
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.11",
     "@shopify/jest-dom-mocks": "^5.0.0",
     "@sindresorhus/tsconfig": "^5.0.0",

playwright.config.ts

@@ -1,18 +1,5 @@
 import { defineConfig } from "@playwright/test";
 import { CI } from "./end-to-end-tests/env";
-import fs from "node:fs";
-import { getAuthProfilePathFile } from "./end-to-end-tests/fixtures/utils";
-
-// Speed up local development by skipping the authentication setup if it's already done.
-// NOTE: You will have to restart the test runner if you need to re-run the auth setup.
-const isAuthSetupDone = (chromiumChannel: "msedge" | "chrome") => {
-  if (CI) {
-    return false;
-  }
-
-  const filePath = getAuthProfilePathFile(chromiumChannel);
-  return fs.existsSync(filePath);
-};
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -34,6 +21,7 @@ export default defineConfig<{ chromiumChannel: string }>({
     /* Timeout for each assertion. If a particular interaction is timing out, adjust its specific timeout value rather than this global setting */
     timeout: 5000,
   },
+  reportSlowTests: null,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: [["html", { outputFolder: "./end-to-end-tests/.report" }]],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
@@ -65,14 +53,15 @@ export default defineConfig<{ chromiumChannel: string }>({
       use: {
         chromiumChannel: "chrome",
       },
-      dependencies: isAuthSetupDone("chrome") ? [] : ["chromeSetup"],
+      // For faster local development, you can filter out the setup project in --ui mode to skip rerunning the setup project
+      dependencies: ["chromeSetup"],
     },
     {
       name: "edge",
       use: {
         chromiumChannel: "msedge",
       },
-      dependencies: isAuthSetupDone("msedge") ? [] : ["edgeSetup"],
+      dependencies: ["edgeSetup"],
     },
   ],
 });