Run full E2E test suite on PRs

.env

@@ -115,8 +115,8 @@ E2E_TEST_ACCOUNT_EMAIL_ZERO_BROKERS=
 E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES_ZERO_BROKERS=
 E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED=
 
-E2E_TEST_PAYPAL_LOGIN =
-E2E_TEST_PAYPAL_PASSWORD =
+E2E_TEST_PAYPAL_LOGIN=
+E2E_TEST_PAYPAL_PASSWORD=
 
 # Monitor Premium features
 # Link to start user on the subscription process. PREMIUM_ENABLED must be set to `true`.

.github/workflows/e2e_cron.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Run Playwright tests - ${{ inputs.environment != null && inputs.environment || 'stage' }}
         if: github.actor != 'dependabot[bot]'
         run: npm run e2e
-        timeout-minutes: 40
+        timeout-minutes: 20
         env:
           E2E_TEST_ENV: ${{ inputs.environment != null && inputs.environment || 'stage' }}
           E2E_TEST_BASE_URL: ${{ secrets.E2E_TEST_BASE_URL }}

.github/workflows/e2e_pr_full.yml

@@ -0,0 +1,99 @@
+name: Monitor E2E Test Suite (full)
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+jobs:
+  e2e-tests-full:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: blurts
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+        - 5432:5432
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20.9.x
+
+      - name: Install dependencies
+        run: npm ci
+      - name: Setting up postgres
+        run: npm run db:migrate
+        env:
+          DATABASE_URL: postgres://postgres:postgres@localhost:5432/blurts
+      - name: Store Playwright's Version
+        run: |
+          # Get the current Playwright version listed in package.json
+          PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
+          echo "Playwright Version: $PLAYWRIGHT_VERSION"
+          echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
+      - name: Cache Playwright Browsers for Playwright's Version
+        id: cache-playwright-browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }}
+
+      - name: Setup Playwright Browser
+        if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps
+
+      - name: Run Playwright tests
+        if: github.actor != 'dependabot[bot]'
+        run: npm run e2e -- --update-snapshots
+        timeout-minutes: 20
+        env:
+          E2E_TEST_ENV: local
+          E2E_TEST_BASE_URL: http://localhost:6060
+          E2E_TEST_ACCOUNT_EMAIL: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL }}
+          E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES }}
+          E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED }}
+          E2E_TEST_ACCOUNT_PASSWORD: ${{ secrets.E2E_TEST_ACCOUNT_PASSWORD }}
+          E2E_TEST_PAYPAL_LOGIN: ${{ secrets.E2E_TEST_PAYPAL_LOGIN }}
+          E2E_TEST_PAYPAL_PASSWORD: ${{ secrets.E2E_TEST_PAYPAL_PASSWORD }}
+          ADMINS: ${{ secrets.ADMINS }}
+          OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_CLIENT_SECRET_LOCAL }}
+          OAUTH_ACCOUNT_URI: ${{ secrets.OAUTH_ACCOUNT_URI }}
+          ONEREP_API_KEY: ${{ secrets.ONEREP_API_KEY }}
+          NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }}
+          NEXTAUTH_URL: ${{ secrets.NEXTAUTH_URL }}
+          DATABASE_URL: postgres://postgres:postgres@localhost:5432/blurts
+          HIBP_KANON_API_TOKEN: ${{ secrets.HIBP_KANON_API_TOKEN }}
+          HIBP_API_TOKEN: ${{ secrets.HIBP_API_TOKEN }}
+          HIBP_KANON_API_ROOT: "http://localhost:6060/api/mock/hibp"
+          ONEREP_API_BASE: "http://localhost:6060/api/mock/onerep/"
+          PREMIUM_PRODUCT_ID: ${{ secrets.STAGE_PREMIUM_PRODUCT_ID }}
+          PREMIUM_PLAN_ID_MONTHLY_US: ${{ secrets.STAGE_PREMIUM_PLAN_ID_MONTHLY_US }}
+          PREMIUM_PLAN_ID_YEARLY_US: ${{ secrets.STAGE_PREMIUM_PLAN_ID_YEARLY_US }}
+          REDIS_URL: "redis://redis.mock"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-results
+          path: src/e2e/test-results/
+          retention-days: 30

.github/workflows/e2e_pr.yml → .github/workflows/e2e_pr_smoke.yml

@@ -1,11 +1,11 @@
-name: Monitor e2e Smoke Tests
+name: Monitor E2E Test Suite (smoke)
 on:
   push:
     branches: [ main ]
   pull_request:
     branches: [ main ]
 jobs:
-  e2e-tests:
+  e2e-tests-smoke:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
@@ -46,7 +46,6 @@ jobs:
           PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
           echo "Playwright Version: $PLAYWRIGHT_VERSION"
           echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
-
       - name: Cache Playwright Browsers for Playwright's Version
         id: cache-playwright-browsers
         uses: actions/cache@v4
@@ -69,7 +68,7 @@ jobs:
           E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES }}
           E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED }}
           E2E_TEST_ACCOUNT_PASSWORD: ${{ secrets.E2E_TEST_ACCOUNT_PASSWORD }}
-          E2E_TEST_PAYPAL_LOGIN:  ${{ secrets.E2E_TEST_PAYPAL_LOGIN }}
+          E2E_TEST_PAYPAL_LOGIN: ${{ secrets.E2E_TEST_PAYPAL_LOGIN }}
           E2E_TEST_PAYPAL_PASSWORD: ${{ secrets.E2E_TEST_PAYPAL_PASSWORD }}
           ADMINS: ${{ secrets.ADMINS }}
           OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_CLIENT_SECRET_LOCAL }}
@@ -82,9 +81,6 @@ jobs:
           HIBP_API_TOKEN: ${{ secrets.HIBP_API_TOKEN }}
           HIBP_KANON_API_ROOT: "http://localhost:6060/api/mock/hibp"
           ONEREP_API_BASE: "http://localhost:6060/api/mock/onerep/"
-          # MNTOR-3516: Our tests are currently set up to expect accounts to act like
-          # old user accounts, so let's pretend they all are:
-          BROKER_SCAN_RELEASE_DATE: "3000-12-31"
           REDIS_URL: "redis://redis.mock"
       - uses: actions/upload-artifact@v4
         if: always()

docs/release_process.md

@@ -74,100 +74,112 @@ set up and changes are ready to be reviewed.
 Every commit to `main` is automatically deployed to the [Stage][stage] server via Github Actions and Jenkins.
 
 ### PR Merges
+
 PRs can only be merged once they pass all the required checks:
-* Lint
-* Build
-* Unit Tests
-* E2E Tests
-* Deploy Previews
+
+- Lint
+- Build
+- Unit Tests
+- E2E Tests
+- Deploy Previews
 
 A PR also needs at least one approval from an ENGR team member to be merged into `main`.
 
 Once a PR is successfully merged:
+
 1. ensure that the merge commit in `main` branch passes all checks and a docker image is successfully deployed.
 2. Jenkins will kick off the deployment of the latest built docker image to stage environment
 3. A webhook will send status messages into the `#fx-monitor-engineering` channel.
-   * Watch for messages: `FX MONITOR STAGE STARTED` and `FX MONITOR STAGE COMPLETE`
+   - Watch for messages: `FX MONITOR STAGE STARTED` and `FX MONITOR STAGE COMPLETE`
 
 ## Release to Production
+
 Before releasing to production, we need to assess the current state of our work on stage. We need to cross-reference what's already on stage and what's been greenlit by QA. To do this, we need to find the difference between what was released last time in production and what we currently have on stage.
 
 ### Check the diff in Release Notes and notify the team
+
 [make a release on GitHub][github-new-release] in order to check the difference.
+
 1. Choose the tag you want for your release. Use today's date (e.g., `2024.09.01`)
 2. Type the same tag name for the release title (e.g., `2024.09.01`)
 3. Click the "Generate release notes" button!
-4. *DO NOT* press "Publish release" yet
+4. _DO NOT_ press "Publish release" yet
 5. Copy and Paste the release notes in the engineering slack channel so the team is aware
 6. Go through the PRs, cross-reference the tickets in the PRs with the [Jira][jira] board to see if QA has approved the tickets. If anything is unclear, make sure to tag the author of the PR.
 7. If anything has not been properly tested, make a note, and again, double check with the person
 8. If everything looks good, proceed to release, otherwise refer to the section `Stage-fixes` below.
 
 ### Update Production Environment Variables
+
 In the cases where we need to update or add new environment variables, we need to get help from SRE:
+
 1. File an [SRE ticket][sre-board] for the env var change.
    - In the title, make sure to mention "Production"
    - Make sure to include the value and the correct variable name
    - Make sure to specify if it's a `secret` or a regular variable
 2. When appropriate, wait for SRE to make the changes before proceeding with the production release.
 
 ### 1-click Production Release
+
 After you push the tag to GitHub, you should also
 [make a release on GitHub][github-new-release] for the tag.
 
 1. After all the checks above look good, click "Publish release"
 2. Go to the `main` branch and make sure all the checks succeeded
 3. Go to [DockerHub][dockerhub] to ensure that a tag with today's date is present.
 4. Run [E2E cron][e2e] against stage (with the latest update)
-    * if there are errors, make sure the cause is understood
-    * fix the e2e errors or change the tests when appropriate before proceeding
-6. Check the stage Sentry and GCP error logs
-7. Run [1-Click Deploy Github Action][1-click deploy]
-   * Click on `Run workflow`
-   * `Branch:main` is selected
-   * `prod` is selected for environment
-   * Input the tag created earlier (today's date, e.g., `2024.09.01`)
-   * Click on `Run workflow` when ready
-8. A webhook will send status messages into the `#fx-monitor-engineering` channel.
-    * Watch for messages: `pushing to production started` and `successfully deployed to production`
-9. After successful deploy, conduct some basic sanity check:
-    * Check sentry prod project for a spike in any new issues
-    * Check [grafana dashboard][grafana-dashboard] for any unexpected spike in ops
-    * Spot-check the site for basic functionality
+   - if there are errors, make sure the cause is understood
+   - fix the e2e errors or change the tests when appropriate before proceeding
+5. Check the stage Sentry and GCP error logs
+6. Run [1-Click Deploy Github Action][1-click deploy]
+   - Click on `Run workflow`
+   - `Branch:main` is selected
+   - `prod` is selected for environment
+   - Input the tag created earlier (today's date, e.g., `2024.09.01`)
+   - Click on `Run workflow` when ready
+7. A webhook will send status messages into the `#fx-monitor-engineering` channel.
+   - Watch for messages: `pushing to production started` and `successfully deployed to production`
+8. After successful deploy, conduct some basic sanity check:
+   - Check sentry prod project for a spike in any new issues
+   - Check [grafana dashboard][grafana-dashboard] for any unexpected spike in ops
+   - Spot-check the site for basic functionality
 
 ### Update Jira
 
 On our [Jira][jira] board, review the tickets listed under "Merged to main." If those were included in the release you just created, drag those tickets to either the "Promoted to Prod" or "Done" column. This will notify QA that they can verify the behavior on Prod if necessary.
 
 If you're unsure whether a ticket was included in the release, ask the assigned person to move it if needed.
+
 ## Stage-fixes
 
-Ideally, every change can ride the regular weekly release "trains". But sometimes, not everything in `main` can go out. Since we've adopted feature flags, these scenarios are becoming rarer. However, we still cannot guarantee that they never happen. 
+Ideally, every change can ride the regular weekly release "trains". But sometimes, not everything in `main` can go out. Since we've adopted feature flags, these scenarios are becoming rarer. However, we still cannot guarantee that they never happen.
 
 Wherever feature flags aren't applicable, there are generally two scenarios we need to consider:
+
 1. If the diff in changes is minimal (eg. can be traced back to a PR or two), the easiest way is to revert
 2. If the diff is not minimal, or a significant portion of the tickets haven't been QA'd:
-   * we can choose to delay the release (ask the team for consensus)
-   * we can create a separate release branch
+   - we can choose to delay the release (ask the team for consensus)
+   - we can create a separate release branch
 
 ### Revert
+
 1. Revert the PR(s)
 2. Create a Github [Release][github-new-release]
 3. Revert the revert after production deployment is successful
-   * After the revert of revert is successfully merged into `main`, stage should be automatically put back to the state before Production release
+   - After the revert of revert is successfully merged into `main`, stage should be automatically put back to the state before Production release
 
 ### Separate release branch
+
 1. Create a branch on top of `main`
 2. Work on taking out the features that should not be included (not feature-flagged)
 3. Create a Github [Release][github-new-release]
-   * In the release, make sure to pick your branch (`main` is default)
-   * Generate the release note, double check and make sure that it makes sense
+   - In the release, make sure to pick your branch (`main` is default)
+   - Generate the release note, double check and make sure that it makes sense
 4. Proceed with the production release
 
-
 ## Future
 
-After adding 1-click production deploy capability and broadly adopting [feature flags][feature-flags], we are looking into ways to increase our production release frequency. The main challenge here is to coordiate our QA effort with our latest stage CICD deployments. 
+After adding 1-click production deploy capability and broadly adopting [feature flags][feature-flags], we are looking into ways to increase our production release frequency. The main challenge here is to coordiate our QA effort with our latest stage CICD deployments.
 
 We are starting to look into creating daily GitHub pre-releases via GHA, and once QA'd, having these deployed automatically or manually by base load engineers.
 

playwright.config.js

@@ -50,11 +50,14 @@ export default defineConfig({
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
 
+  /* Limit the number of failures */
+  maxFailures: 1,
+
   /* Retry on CI only */
   retries: process.env.CI ? 1 : 0,
 
-  /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
+  /* Use a custom percentage of available wokers in CI and use default locally. */
+  workers: process.env.CI ? "75%" : undefined,
 
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: process.env.CI ? [['github'], ['html']] : 'html',
@@ -65,8 +68,7 @@ export default defineConfig({
     actionTimeout: 0,
 
     /* Base URL to use in actions like `await page.goto('/')`. */
-    baseURL: process.env.E2E_TEST_BASE_URL ?? 'https://stage.firefoxmonitor.nonprod.cloudops.mozgcp.net',
-    // baseURL: 'http://localhost:6060',
+    baseURL: process.env.E2E_TEST_BASE_URL,
 
     /* automatically take screenshot only on failures */
     screenshot: 'only-on-failure',

src/app/functions/server/isPrePlusUser.ts

@@ -3,8 +3,8 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 import { Session } from "next-auth";
+import { isPrePlusDate } from "../universal/isPrePlusDate";
 import "./notInClientComponent";
-import { parseIso8601Datetime } from "../../../utils/parse";
 
 /**
  * Determine whether the user's account predates Monitor Plus
@@ -20,22 +20,8 @@ export function isPrePlusUser(user: Session["user"]): boolean {
     return false;
   }
 
-  const brokerScanReleaseDateParts = (
-    process.env.BROKER_SCAN_RELEASE_DATE ?? ""
-  ).split("-");
-  if (brokerScanReleaseDateParts[0] === "") {
-    brokerScanReleaseDateParts[0] = "2023";
-  }
-  const brokerScanReleaseDate = new Date(
-    Date.UTC(
-      Number.parseInt(brokerScanReleaseDateParts[0], 10),
-      Number.parseInt(brokerScanReleaseDateParts[1] ?? "12", 10) - 1,
-      Number.parseInt(brokerScanReleaseDateParts[2] ?? "05", 10),
-    ),
-  );
-
-  return (
-    (parseIso8601Datetime(user.subscriber.created_at)?.getTime() ?? 0) <
-    brokerScanReleaseDate.getTime()
+  return isPrePlusDate(
+    process.env.BROKER_SCAN_RELEASE_DATE ?? "",
+    user.subscriber.created_at,
   );
 }

src/app/functions/universal/isPrePlusDate.ts

@@ -0,0 +1,27 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { parseIso8601Datetime } from "../../../utils/parse";
+
+export function isPrePlusDate(
+  plusReleaseDateString: string,
+  dateStringToCompare: string,
+) {
+  const brokerScanReleaseDateParts = plusReleaseDateString.split("-");
+  if (brokerScanReleaseDateParts[0] === "") {
+    brokerScanReleaseDateParts[0] = "2023";
+  }
+  const brokerScanReleaseDate = new Date(
+    Date.UTC(
+      Number.parseInt(brokerScanReleaseDateParts[0], 10),
+      Number.parseInt(brokerScanReleaseDateParts[1] ?? "12", 10) - 1,
+      Number.parseInt(brokerScanReleaseDateParts[2] ?? "05", 10),
+    ),
+  );
+
+  return (
+    (parseIso8601Datetime(dateStringToCompare)?.getTime() ?? 0) <
+    brokerScanReleaseDate.getTime()
+  );
+}