Setup Cloud e2e test framework and write first tests

.github/workflows/web-test-e2e.yml

@@ -2,7 +2,7 @@ name: End-to-end tests of web+runtime
 on:
   pull_request:
     paths:
-      - ".github/workflows/web-test.yml"
+      - ".github/workflows/web-test-e2e.yml"
       - "admin/**"
       - "cli/**"
       - "runtime/**"
@@ -13,6 +13,9 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -23,25 +26,45 @@ jobs:
         with:
           filters: |
             admin:
-              - ".github/workflows/web-test.yml"
+              - ".github/workflows/web-test-e2e.yml"
               - "admin/**"
               - "cli/**"
               - "runtime/**"
               - "web-admin/**"
             auth:
-              - ".github/workflows/web-test.yml"
+              - ".github/workflows/web-test-e2e.yml"
               - "web-auth/**"
             local:
-              - ".github/workflows/web-test.yml"
+              - ".github/workflows/web-test-e2e.yml"
               - "cli/**"
               - "runtime/**"
               - "web-local/**"
             common:
-              - ".github/workflows/web-test.yml"
+              - ".github/workflows/web-test-e2e.yml"
               - "cli/**"
               - "runtime/**"
               - "web-common/**"
 
+      - name: Evaluate e2e tests matrix
+        id: matrix
+        run: |
+          echo "matrix=$(jq -n '[
+              "web-admin" | select(env.FILTER_ADMIN == "true" or env.FILTER_COMMON == "true"),
+              "web-local" | select(env.FILTER_LOCAL == "true" or env.FILTER_COMMON == "true")
+            ]' | jq -c)" >> "$GITHUB_OUTPUT"
+        env:
+          FILTER_ADMIN: ${{ steps.filter.outputs.admin }}
+          FILTER_COMMON: ${{ steps.filter.outputs.common }}
+          FILTER_LOCAL: ${{ steps.filter.outputs.local }}
+
+  e2e:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        name: ${{ fromJson(needs.build.outputs.matrix) }}
+    steps:
       - name: Set up NodeJS
         uses: actions/setup-node@v4
         with:
@@ -51,6 +74,10 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: 1.23
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: go build cache
         uses: actions/cache@v4
         with:
@@ -70,21 +97,30 @@ jobs:
       - name: Build and embed static UI
         run: PLAYWRIGHT_TEST=true make cli
 
+      - name: Add CLI binary to PATH
+        run: echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH
+
       - name: Install browser for UI tests
         run: npx playwright install
 
-      - name: Test web local
-        if: ${{ steps.filter.outputs.local == 'true' || steps.filter.outputs.common == 'true' }}
-        run: npm run test -w web-local
+      - name: Authenticate GCloud
+        uses: google-github-actions/auth@v2
+        if: matrix.name == 'web-admin'
+        with:
+          credentials_json: "${{ secrets.RILL_BINARY_SA }}"
+
+      - name: Set up GCloud SDK
+        if: matrix.name == 'web-admin'
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Test ${{ matrix.name }}
+        run: |-
+          npm run build -w ${{ matrix.name }}
+          npm run test -w ${{ matrix.name }}
 
       - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: playwright-report
-          path: web-local/playwright-report/
+          path: ${{ matrix.name }}/playwright-report/
           retention-days: 30
-
-      - name: Build web admin
-        if: ${{ steps.filter.outputs.admin == 'true' || steps.filter.outputs.common == 'true' }}
-        run: |-
-          npm run build -w web-admin

.gitignore

@@ -26,7 +26,8 @@ tsc-tmp/
 /dev-project*
 /dev-cloud-state*
 /e2e-cloud-state*
-/playwright-report
+**/playwright-report
+**/test-results
 
 # data files
 *.duckdb

web-admin/.gitignore

@@ -1,2 +1,3 @@
 /.svelte-kit
-build
+build
+playwright

web-admin/package.json

@@ -15,7 +15,7 @@
   },
   "devDependencies": {
     "@fontsource/fira-mono": "^4.5.0",
-    "@playwright/test": "^1.33.0",
+    "@playwright/test": "^1.49.1",
     "@rilldata/svelte-query": "^4.29.20-0.0.3",
     "@sveltejs/adapter-static": "^3.0.5",
     "@sveltejs/kit": "^2.7.1",

web-admin/playwright.config.ts

@@ -1,10 +1,43 @@
-import type { PlaywrightTestConfig } from "@playwright/test";
+import { devices, type PlaywrightTestConfig } from "@playwright/test";
+import dotenv from "dotenv";
+import path from "path";
+import { fileURLToPath } from "url";
+import { ADMIN_AUTH_FILE } from "./tests/setup/constants";
+
+// Load environment variables from our root `.env` file
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+dotenv.config({ path: path.resolve(__dirname, "../.env") });
 
 const config: PlaywrightTestConfig = {
+  globalSetup: "./tests/setup/globalSetup.ts",
+  globalTeardown: "./tests/setup/globalTeardown.ts",
   webServer: {
     command: "npm run build && npm run preview",
-    port: 4173,
+    port: 3000,
+    reuseExistingServer: !process.env.CI,
+    timeout: 120_000,
+  },
+  retries: 0,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  use: {
+    baseURL: "http://localhost:3000",
+    ...devices["Desktop Chrome"],
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+    video: "retain-on-failure",
   },
+  projects: [
+    { name: "bootstrap", testMatch: "bootstrap.spec.ts" },
+    {
+      name: "e2e",
+      dependencies: process.env.E2E_SKIP_BOOTSTRAP ? [] : ["bootstrap"],
+      testIgnore: "/setup",
+      use: {
+        storageState: ADMIN_AUTH_FILE,
+      },
+    },
+  ],
 };
 
 export default config;

web-admin/tests/explores.spec.ts

@@ -0,0 +1,19 @@
+import { expect } from "@playwright/test";
+import { test } from "./setup/base";
+
+test.describe("Explores", () => {
+  test("should have data", async ({ page }) => {
+    await page.goto("/e2e/openrtb");
+
+    // Navigate to the explore
+    await page
+      .getByRole("link", { name: "Programmatic Ads Auction" })
+      .first()
+      .click();
+
+    // Check the Big Number
+    await expect(
+      page.getByRole("button", { name: "Requests 635M" }),
+    ).toBeVisible();
+  });
+});

web-admin/tests/homepage.spec.ts

@@ -0,0 +1,16 @@
+import { expect } from "@playwright/test";
+import { test } from "./setup/base";
+
+test.describe("Homepage", () => {
+  test("Authenticated user should see the homepage", async ({ page }) => {
+    await page.goto("/");
+    await expect(page.getByText("Hi [email protected]!")).toBeVisible();
+  });
+
+  test("Unauthenticated user should be redirected to login", async ({
+    anonPage,
+  }) => {
+    await anonPage.goto("/");
+    await expect(anonPage.getByText("Log in to Rill")).toBeVisible();
+  });
+});

web-admin/tests/organizations.spec.ts

@@ -0,0 +1,12 @@
+import { expect } from "@playwright/test";
+import { test } from "./setup/base";
+
+test.describe("Organizations", () => {
+  test("admins should see the admin-only pages", async ({ adminPage }) => {
+    await adminPage.goto("/e2e");
+    await expect(adminPage.getByRole("link", { name: "Users" })).toBeVisible();
+    await expect(
+      adminPage.getByRole("link", { name: "Settings" }),
+    ).toBeVisible();
+  });
+});

web-admin/tests/projects.spec.ts

@@ -0,0 +1,12 @@
+import { expect } from "@playwright/test";
+import { test } from "./setup/base";
+
+test.describe("Projects", () => {
+  test("admins should see the admin-only pages", async ({ adminPage }) => {
+    await adminPage.goto("/e2e/openrtb");
+    await expect(adminPage.getByRole("link", { name: "Status" })).toBeVisible();
+    await expect(
+      adminPage.getByRole("link", { name: "Settings" }),
+    ).toBeVisible();
+  });
+});

web-admin/tests/setup/base.ts

@@ -0,0 +1,47 @@
+import { test as base, type Page } from "@playwright/test";
+import { ADMIN_AUTH_FILE, VIEWER_AUTH_FILE } from "./constants";
+import { cliLogin, cliLogout } from "./fixtures/cli";
+
+type MyFixtures = {
+  adminPage: Page;
+  viewerPage: Page;
+  anonPage: Page;
+  cli: void;
+};
+
+export const test = base.extend<MyFixtures>({
+  // Note: the `e2e` project uses the admin auth file by default, so it's likely that
+  // this fixture won't be used often.
+  adminPage: async ({ browser }, use) => {
+    const context = await browser.newContext({
+      storageState: ADMIN_AUTH_FILE,
+    });
+    const adminPage = await context.newPage();
+    await use(adminPage);
+    await context.close();
+  },
+
+  viewerPage: async ({ browser }, use) => {
+    const context = await browser.newContext({
+      storageState: VIEWER_AUTH_FILE,
+    });
+    const viewerPage = await context.newPage();
+    await use(viewerPage);
+    await context.close();
+  },
+
+  anonPage: async ({ browser }, use) => {
+    const context = await browser.newContext({
+      storageState: { cookies: [], origins: [] },
+    });
+    const anonPage = await context.newPage();
+    await use(anonPage);
+    await context.close();
+  },
+
+  cli: async ({ page }, use) => {
+    await cliLogin(page);
+    await use();
+    await cliLogout();
+  },
+});