Skip to content

Commit

Permalink
Merge pull request #4996 from mozilla/e2e-convert-cron-to-pr
Browse files Browse the repository at this point in the history
Run full E2E test suite on PRs
  • Loading branch information
flozia authored Sep 26, 2024
2 parents f086c5c + a1d45f4 commit 9305737
Show file tree
Hide file tree
Showing 16 changed files with 268 additions and 124 deletions.
4 changes: 2 additions & 2 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ E2E_TEST_ACCOUNT_EMAIL_ZERO_BROKERS=
E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES_ZERO_BROKERS=
E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED=

E2E_TEST_PAYPAL_LOGIN =
E2E_TEST_PAYPAL_PASSWORD =
E2E_TEST_PAYPAL_LOGIN=
E2E_TEST_PAYPAL_PASSWORD=

# Monitor Premium features
# Link to start user on the subscription process. PREMIUM_ENABLED must be set to `true`.
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e_cron.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
- name: Run Playwright tests - ${{ inputs.environment != null && inputs.environment || 'stage' }}
if: github.actor != 'dependabot[bot]'
run: npm run e2e
timeout-minutes: 40
timeout-minutes: 20
env:
E2E_TEST_ENV: ${{ inputs.environment != null && inputs.environment || 'stage' }}
E2E_TEST_BASE_URL: ${{ secrets.E2E_TEST_BASE_URL }}
Expand Down
99 changes: 99 additions & 0 deletions .github/workflows/e2e_pr_full.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
name: Monitor E2E Test Suite (full)
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
e2e-tests-full:
timeout-minutes: 60
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres
# Provide the password for postgres
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: blurts
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432

steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20.9.x

- name: Install dependencies
run: npm ci
- name: Setting up postgres
run: npm run db:migrate
env:
DATABASE_URL: postgres://postgres:postgres@localhost:5432/blurts
- name: Store Playwright's Version
run: |
# Get the current Playwright version listed in package.json
PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
echo "Playwright Version: $PLAYWRIGHT_VERSION"
echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
- name: Cache Playwright Browsers for Playwright's Version
id: cache-playwright-browsers
uses: actions/cache@v4
with:
path: ~/.cache/ms-playwright
key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }}

- name: Setup Playwright Browser
if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
run: npx playwright install --with-deps

- name: Run Playwright tests
if: github.actor != 'dependabot[bot]'
run: npm run e2e -- --update-snapshots
timeout-minutes: 20
env:
E2E_TEST_ENV: local
E2E_TEST_BASE_URL: http://localhost:6060
E2E_TEST_ACCOUNT_EMAIL: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL }}
E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES }}
E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED }}
E2E_TEST_ACCOUNT_PASSWORD: ${{ secrets.E2E_TEST_ACCOUNT_PASSWORD }}
E2E_TEST_PAYPAL_LOGIN: ${{ secrets.E2E_TEST_PAYPAL_LOGIN }}
E2E_TEST_PAYPAL_PASSWORD: ${{ secrets.E2E_TEST_PAYPAL_PASSWORD }}
ADMINS: ${{ secrets.ADMINS }}
OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_CLIENT_SECRET_LOCAL }}
OAUTH_ACCOUNT_URI: ${{ secrets.OAUTH_ACCOUNT_URI }}
ONEREP_API_KEY: ${{ secrets.ONEREP_API_KEY }}
NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }}
NEXTAUTH_URL: ${{ secrets.NEXTAUTH_URL }}
DATABASE_URL: postgres://postgres:postgres@localhost:5432/blurts
HIBP_KANON_API_TOKEN: ${{ secrets.HIBP_KANON_API_TOKEN }}
HIBP_API_TOKEN: ${{ secrets.HIBP_API_TOKEN }}
HIBP_KANON_API_ROOT: "http://localhost:6060/api/mock/hibp"
ONEREP_API_BASE: "http://localhost:6060/api/mock/onerep/"
PREMIUM_PRODUCT_ID: ${{ secrets.STAGE_PREMIUM_PRODUCT_ID }}
PREMIUM_PLAN_ID_MONTHLY_US: ${{ secrets.STAGE_PREMIUM_PLAN_ID_MONTHLY_US }}
PREMIUM_PLAN_ID_YEARLY_US: ${{ secrets.STAGE_PREMIUM_PLAN_ID_YEARLY_US }}
REDIS_URL: "redis://redis.mock"
- uses: actions/upload-artifact@v4
if: always()
with:
name: playwright-report
path: playwright-report/
retention-days: 30
- uses: actions/upload-artifact@v4
if: always()
with:
name: test-results
path: src/e2e/test-results/
retention-days: 30
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
name: Monitor e2e Smoke Tests
name: Monitor E2E Test Suite (smoke)
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
e2e-tests:
e2e-tests-smoke:
timeout-minutes: 60
runs-on: ubuntu-latest
# Service containers to run with `container-job`
Expand Down Expand Up @@ -46,7 +46,6 @@ jobs:
PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
echo "Playwright Version: $PLAYWRIGHT_VERSION"
echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
- name: Cache Playwright Browsers for Playwright's Version
id: cache-playwright-browsers
uses: actions/cache@v4
Expand All @@ -69,7 +68,7 @@ jobs:
E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_ZERO_BREACHES }}
E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED: ${{ secrets.E2E_TEST_ACCOUNT_EMAIL_EXPOSURES_STARTED }}
E2E_TEST_ACCOUNT_PASSWORD: ${{ secrets.E2E_TEST_ACCOUNT_PASSWORD }}
E2E_TEST_PAYPAL_LOGIN: ${{ secrets.E2E_TEST_PAYPAL_LOGIN }}
E2E_TEST_PAYPAL_LOGIN: ${{ secrets.E2E_TEST_PAYPAL_LOGIN }}
E2E_TEST_PAYPAL_PASSWORD: ${{ secrets.E2E_TEST_PAYPAL_PASSWORD }}
ADMINS: ${{ secrets.ADMINS }}
OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_CLIENT_SECRET_LOCAL }}
Expand All @@ -82,9 +81,6 @@ jobs:
HIBP_API_TOKEN: ${{ secrets.HIBP_API_TOKEN }}
HIBP_KANON_API_ROOT: "http://localhost:6060/api/mock/hibp"
ONEREP_API_BASE: "http://localhost:6060/api/mock/onerep/"
# MNTOR-3516: Our tests are currently set up to expect accounts to act like
# old user accounts, so let's pretend they all are:
BROKER_SCAN_RELEASE_DATE: "3000-12-31"
REDIS_URL: "redis://redis.mock"
- uses: actions/upload-artifact@v4
if: always()
Expand Down
72 changes: 42 additions & 30 deletions docs/release_process.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,100 +74,112 @@ set up and changes are ready to be reviewed.
Every commit to `main` is automatically deployed to the [Stage][stage] server via Github Actions and Jenkins.

### PR Merges

PRs can only be merged once they pass all the required checks:
* Lint
* Build
* Unit Tests
* E2E Tests
* Deploy Previews

- Lint
- Build
- Unit Tests
- E2E Tests
- Deploy Previews

A PR also needs at least one approval from an ENGR team member to be merged into `main`.

Once a PR is successfully merged:

1. ensure that the merge commit in `main` branch passes all checks and a docker image is successfully deployed.
2. Jenkins will kick off the deployment of the latest built docker image to stage environment
3. A webhook will send status messages into the `#fx-monitor-engineering` channel.
* Watch for messages: `FX MONITOR STAGE STARTED` and `FX MONITOR STAGE COMPLETE`
- Watch for messages: `FX MONITOR STAGE STARTED` and `FX MONITOR STAGE COMPLETE`

## Release to Production

Before releasing to production, we need to assess the current state of our work on stage. We need to cross-reference what's already on stage and what's been greenlit by QA. To do this, we need to find the difference between what was released last time in production and what we currently have on stage.

### Check the diff in Release Notes and notify the team

[make a release on GitHub][github-new-release] in order to check the difference.

1. Choose the tag you want for your release. Use today's date (e.g., `2024.09.01`)
2. Type the same tag name for the release title (e.g., `2024.09.01`)
3. Click the "Generate release notes" button!
4. *DO NOT* press "Publish release" yet
4. _DO NOT_ press "Publish release" yet
5. Copy and Paste the release notes in the engineering slack channel so the team is aware
6. Go through the PRs, cross-reference the tickets in the PRs with the [Jira][jira] board to see if QA has approved the tickets. If anything is unclear, make sure to tag the author of the PR.
7. If anything has not been properly tested, make a note, and again, double check with the person
8. If everything looks good, proceed to release, otherwise refer to the section `Stage-fixes` below.

### Update Production Environment Variables

In the cases where we need to update or add new environment variables, we need to get help from SRE:

1. File an [SRE ticket][sre-board] for the env var change.
- In the title, make sure to mention "Production"
- Make sure to include the value and the correct variable name
- Make sure to specify if it's a `secret` or a regular variable
2. When appropriate, wait for SRE to make the changes before proceeding with the production release.

### 1-click Production Release

After you push the tag to GitHub, you should also
[make a release on GitHub][github-new-release] for the tag.

1. After all the checks above look good, click "Publish release"
2. Go to the `main` branch and make sure all the checks succeeded
3. Go to [DockerHub][dockerhub] to ensure that a tag with today's date is present.
4. Run [E2E cron][e2e] against stage (with the latest update)
* if there are errors, make sure the cause is understood
* fix the e2e errors or change the tests when appropriate before proceeding
6. Check the stage Sentry and GCP error logs
7. Run [1-Click Deploy Github Action][1-click deploy]
* Click on `Run workflow`
* `Branch:main` is selected
* `prod` is selected for environment
* Input the tag created earlier (today's date, e.g., `2024.09.01`)
* Click on `Run workflow` when ready
8. A webhook will send status messages into the `#fx-monitor-engineering` channel.
* Watch for messages: `pushing to production started` and `successfully deployed to production`
9. After successful deploy, conduct some basic sanity check:
* Check sentry prod project for a spike in any new issues
* Check [grafana dashboard][grafana-dashboard] for any unexpected spike in ops
* Spot-check the site for basic functionality
- if there are errors, make sure the cause is understood
- fix the e2e errors or change the tests when appropriate before proceeding
5. Check the stage Sentry and GCP error logs
6. Run [1-Click Deploy Github Action][1-click deploy]
- Click on `Run workflow`
- `Branch:main` is selected
- `prod` is selected for environment
- Input the tag created earlier (today's date, e.g., `2024.09.01`)
- Click on `Run workflow` when ready
7. A webhook will send status messages into the `#fx-monitor-engineering` channel.
- Watch for messages: `pushing to production started` and `successfully deployed to production`
8. After successful deploy, conduct some basic sanity check:
- Check sentry prod project for a spike in any new issues
- Check [grafana dashboard][grafana-dashboard] for any unexpected spike in ops
- Spot-check the site for basic functionality

### Update Jira

On our [Jira][jira] board, review the tickets listed under "Merged to main." If those were included in the release you just created, drag those tickets to either the "Promoted to Prod" or "Done" column. This will notify QA that they can verify the behavior on Prod if necessary.

If you're unsure whether a ticket was included in the release, ask the assigned person to move it if needed.

## Stage-fixes

Ideally, every change can ride the regular weekly release "trains". But sometimes, not everything in `main` can go out. Since we've adopted feature flags, these scenarios are becoming rarer. However, we still cannot guarantee that they never happen.
Ideally, every change can ride the regular weekly release "trains". But sometimes, not everything in `main` can go out. Since we've adopted feature flags, these scenarios are becoming rarer. However, we still cannot guarantee that they never happen.

Wherever feature flags aren't applicable, there are generally two scenarios we need to consider:

1. If the diff in changes is minimal (eg. can be traced back to a PR or two), the easiest way is to revert
2. If the diff is not minimal, or a significant portion of the tickets haven't been QA'd:
* we can choose to delay the release (ask the team for consensus)
* we can create a separate release branch
- we can choose to delay the release (ask the team for consensus)
- we can create a separate release branch

### Revert

1. Revert the PR(s)
2. Create a Github [Release][github-new-release]
3. Revert the revert after production deployment is successful
* After the revert of revert is successfully merged into `main`, stage should be automatically put back to the state before Production release
- After the revert of revert is successfully merged into `main`, stage should be automatically put back to the state before Production release

### Separate release branch

1. Create a branch on top of `main`
2. Work on taking out the features that should not be included (not feature-flagged)
3. Create a Github [Release][github-new-release]
* In the release, make sure to pick your branch (`main` is default)
* Generate the release note, double check and make sure that it makes sense
- In the release, make sure to pick your branch (`main` is default)
- Generate the release note, double check and make sure that it makes sense
4. Proceed with the production release


## Future

After adding 1-click production deploy capability and broadly adopting [feature flags][feature-flags], we are looking into ways to increase our production release frequency. The main challenge here is to coordiate our QA effort with our latest stage CICD deployments.
After adding 1-click production deploy capability and broadly adopting [feature flags][feature-flags], we are looking into ways to increase our production release frequency. The main challenge here is to coordiate our QA effort with our latest stage CICD deployments.

We are starting to look into creating daily GitHub pre-releases via GHA, and once QA'd, having these deployed automatically or manually by base load engineers.

Expand Down
10 changes: 6 additions & 4 deletions playwright.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,14 @@ export default defineConfig({
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,

/* Limit the number of failures */
maxFailures: 1,

/* Retry on CI only */
retries: process.env.CI ? 1 : 0,

/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : undefined,
/* Use a custom percentage of available wokers in CI and use default locally. */
workers: process.env.CI ? "75%" : undefined,

/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: process.env.CI ? [['github'], ['html']] : 'html',
Expand All @@ -65,8 +68,7 @@ export default defineConfig({
actionTimeout: 0,

/* Base URL to use in actions like `await page.goto('/')`. */
baseURL: process.env.E2E_TEST_BASE_URL ?? 'https://stage.firefoxmonitor.nonprod.cloudops.mozgcp.net',
// baseURL: 'http://localhost:6060',
baseURL: process.env.E2E_TEST_BASE_URL,

/* automatically take screenshot only on failures */
screenshot: 'only-on-failure',
Expand Down
22 changes: 4 additions & 18 deletions src/app/functions/server/isPrePlusUser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { Session } from "next-auth";
import { isPrePlusDate } from "../universal/isPrePlusDate";
import "./notInClientComponent";
import { parseIso8601Datetime } from "../../../utils/parse";

/**
* Determine whether the user's account predates Monitor Plus
Expand All @@ -20,22 +20,8 @@ export function isPrePlusUser(user: Session["user"]): boolean {
return false;
}

const brokerScanReleaseDateParts = (
process.env.BROKER_SCAN_RELEASE_DATE ?? ""
).split("-");
if (brokerScanReleaseDateParts[0] === "") {
brokerScanReleaseDateParts[0] = "2023";
}
const brokerScanReleaseDate = new Date(
Date.UTC(
Number.parseInt(brokerScanReleaseDateParts[0], 10),
Number.parseInt(brokerScanReleaseDateParts[1] ?? "12", 10) - 1,
Number.parseInt(brokerScanReleaseDateParts[2] ?? "05", 10),
),
);

return (
(parseIso8601Datetime(user.subscriber.created_at)?.getTime() ?? 0) <
brokerScanReleaseDate.getTime()
return isPrePlusDate(
process.env.BROKER_SCAN_RELEASE_DATE ?? "",
user.subscriber.created_at,
);
}
27 changes: 27 additions & 0 deletions src/app/functions/universal/isPrePlusDate.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { parseIso8601Datetime } from "../../../utils/parse";

export function isPrePlusDate(
plusReleaseDateString: string,
dateStringToCompare: string,
) {
const brokerScanReleaseDateParts = plusReleaseDateString.split("-");
if (brokerScanReleaseDateParts[0] === "") {
brokerScanReleaseDateParts[0] = "2023";
}
const brokerScanReleaseDate = new Date(
Date.UTC(
Number.parseInt(brokerScanReleaseDateParts[0], 10),
Number.parseInt(brokerScanReleaseDateParts[1] ?? "12", 10) - 1,
Number.parseInt(brokerScanReleaseDateParts[2] ?? "05", 10),
),
);

return (
(parseIso8601Datetime(dateStringToCompare)?.getTime() ?? 0) <
brokerScanReleaseDate.getTime()
);
}
Loading

0 comments on commit 9305737

Please sign in to comment.