C3: e2es pages tests additions and fixes (#4015)

* C3: remove comment about docusaurus and gatsby

* C3: add docusaurus e2e test

* C3: add gatsby e2e test

* add --no-git-init test flag to remix

* add log to show what C3 command gets run during e2e testing

* set up git user in C3 e2e workflows

* tweak npm_config_user_agent env variable for testing

* increase timeout to 5 mins

* increase prompt delay by a half second

* Add per-framework timeouts in config. Quarantine gatsby

* Prevent promptHandlers from being consumed

* use blog template in astro C3 e2e test

---------

Co-authored-by: James Culveyhouse <[email protected]>

.github/workflows/test-c3.yml

@@ -79,6 +79,12 @@ jobs:
         with:
           bun-version: ${{ env.bun-version }}
 
+      # Needed because gatsby requires git
+      - name: Configure Git
+        run: |
+          git config --global user.email [email protected]
+          git config --global user.name 'Wrangler automated PR updater'
+
       - name: E2E Tests
         run: pnpm run --filter create-cloudflare test:e2e:${{matrix.pm}}
         env:

packages/create-cloudflare/e2e-tests/helpers.ts

@@ -5,6 +5,7 @@ import { join } from "path";
 import { spawn } from "cross-spawn";
 import { sleep } from "helpers/common";
 import { spinnerFrames } from "helpers/interactive";
+import { detectPackageManager } from "helpers/packages";
 import type { SpinnerStyle } from "helpers/interactive";
 
 export const C3_E2E_PREFIX = "c3-e2e-";
@@ -39,7 +40,20 @@ export const runC3 = async ({
 	promptHandlers = [],
 	outputPrefix = "",
 }: RunnerConfig) => {
-	const proc = spawn("node", ["./dist/cli.js", ...argv]);
+	const cmd = "node";
+	const args = ["./dist/cli.js", ...argv];
+	const proc = spawn(cmd, args);
+
+	promptHandlers = [...promptHandlers];
+
+	const { name: pm } = detectPackageManager();
+
+	console.log(
+		`\x1b[44m${outputPrefix} Running C3 with command: \`${cmd} ${args.join(
+			" "
+		)}\` (using ${pm})\x1b[0m`
+	);
+
 	const stdout: string[] = [];
 	const stderr: string[] = [];
 
@@ -49,15 +63,14 @@ export const runC3 = async ({
 			const currentDialog = promptHandlers[0];
 
 			lines.forEach(async (line) => {
-				// Uncomment to debug test output
 				if (filterLine(line)) {
 					console.log(`${outputPrefix} ${line}`);
 				}
 				stdout.push(line);
 
 				if (currentDialog && currentDialog.matcher.test(line)) {
 					// Add a small sleep to avoid input race
-					await sleep(500);
+					await sleep(1000);
 
 					currentDialog.input.forEach((keystroke) => {
 						proc.stdin.write(keystroke);
@@ -116,7 +129,10 @@ export const testProjectDir = (suite: string) => {
 	const clean = (suffix: string) => {
 		const path = getPath(suffix);
 		if (existsSync(path)) {
-			rmSync(path, { recursive: true, force: true });
+			rmSync(path, {
+				recursive: true,
+				force: true,
+			});
 		}
 	};
 

packages/create-cloudflare/e2e-tests/pages.test.ts

@@ -5,32 +5,42 @@ import { fetch } from "undici";
 import { describe, expect, test, afterEach, beforeEach } from "vitest";
 import { version } from "../package.json";
 import { deleteProject } from "../scripts/e2eCleanup";
+import { frameworkCliMap } from "../src/frameworks/package.json";
 import { frameworkToTest } from "./frameworkToTest";
 import { isQuarantineMode, keys, runC3, testProjectDir } from "./helpers";
 import type { RunnerConfig } from "./helpers";
-
-/*
-Areas for future improvement:
-- Add support for frameworks with global installs (like docusaurus, gatsby, etc)
-*/
+import type { TestContext } from "vitest";
 
 const TEST_TIMEOUT = 1000 * 60 * 3;
 
+const frameworks = Object.keys(frameworkCliMap);
+
 type FrameworkTestConfig = RunnerConfig & {
+	timeout?: number;
 	expectResponseToContain: string;
 	testCommitMessage: boolean;
 };
 
 describe.concurrent(`E2E: Web frameworks`, () => {
 	const { getPath, getName, clean } = testProjectDir("pages");
 
+	const getFrameworkNameFromTestContext = (ctx: TestContext) => {
+		const framework = ctx.meta.name.replace(/^Quarantined: /, "");
+		if (!frameworks.includes(framework)) {
+			throw new Error(
+				`Error: Invalid test name, could not detect current framework from it`
+			);
+		}
+		return framework;
+	};
+
 	beforeEach((ctx) => {
-		const framework = ctx.meta.name;
+		const framework = getFrameworkNameFromTestContext(ctx);
 		clean(framework);
 	});
 
 	afterEach(async (ctx) => {
-		const framework = ctx.meta.name;
+		const framework = getFrameworkNameFromTestContext(ctx);
 		clean(framework);
 
 		// Cleanup the pages project in case we need to retry it
@@ -57,6 +67,7 @@ describe.concurrent(`E2E: Web frameworks`, () => {
 			framework,
 			"--deploy",
 			"--no-open",
+			"--no-git",
 		];
 
 		args.push(...argv);
@@ -108,7 +119,7 @@ describe.concurrent(`E2E: Web frameworks`, () => {
 		const { output } = await runCli(framework, {
 			overrides,
 			promptHandlers,
-			argv: [...(argv ?? []), "--deploy", "--no-git"],
+			argv: [...(argv ?? [])],
 		});
 
 		// Verify deployment
@@ -140,9 +151,25 @@ describe.concurrent(`E2E: Web frameworks`, () => {
 	// These are ordered based on speed and reliability for ease of debugging
 	const frameworkTests: Record<string, FrameworkTestConfig> = {
 		astro: {
-			expectResponseToContain: "Welcome to Astro",
+			expectResponseToContain: "Hello, Astronaut!",
+			testCommitMessage: true,
+		},
+		docusaurus: {
+			expectResponseToContain: "Dinosaurs are cool",
 			testCommitMessage: true,
 		},
+		gatsby: {
+			quarantine: true,
+			expectResponseToContain: "Gatsby!",
+			promptHandlers: [
+				{
+					matcher: /Would you like to use a template\?/,
+					input: ["n"],
+				},
+			],
+			testCommitMessage: true,
+			timeout: 1000 * 60 * 6,
+		},
 		hono: {
 			expectResponseToContain: "Hello Hono!",
 			testCommitMessage: false,
@@ -242,7 +269,7 @@ describe.concurrent(`E2E: Web frameworks`, () => {
 					frameworkTests[framework].testCommitMessage
 				);
 			},
-			{ retry: 3, timeout: TEST_TIMEOUT }
+			{ retry: 3, timeout: config.timeout || TEST_TIMEOUT }
 		);
 	});
 

packages/create-cloudflare/src/frameworks/astro/index.ts

@@ -49,7 +49,7 @@ const config: FrameworkConfig = {
 		"--no-install",
 		"--no-git",
 		"--template",
-		"basics",
+		"blog",
 		"--typescript",
 		"strict",
 	],

packages/create-cloudflare/src/frameworks/remix/index.ts

@@ -24,6 +24,6 @@ const config: FrameworkConfig = {
 		"pages:deploy": `${npm} run build && wrangler pages deploy ./public`,
 	},
 	devCommand: "dev",
-	testFlags: ["--typescript", "--no-install"],
+	testFlags: ["--typescript", "--no-install", "--no-git-init"],
 };
 export default config;

packages/create-cloudflare/src/helpers/packages.ts

@@ -19,10 +19,12 @@ export const detectPackageManager = () => {
 			case "pnpm":
 				name = "pnpm";
 				version = devDependencies["pnpm"].replace("^", "");
+				process.env.npm_config_user_agent = "pnpm";
 				break;
 			case "yarn":
 				name = "yarn";
 				version = devDependencies["yarn"].replace("^", "");
+				process.env.npm_config_user_agent = "yarn";
 				break;
 			case "bun":
 				name = "bun";
@@ -31,6 +33,7 @@ export const detectPackageManager = () => {
 			case "npm":
 				name = "npm";
 				version = "0.0.0";
+				process.env.npm_config_user_agent = "npm";
 				break;
 		}
 	}

packages/create-cloudflare/turbo.json

@@ -4,7 +4,7 @@
 
 	"pipeline": {
 		"build": {
-			"env": ["VITEST", "TEST_PM", "CI"]
+			"env": ["VITEST", "TEST_PM", "npm_config_user_agent", "CI"]
 		},
 		"test:e2e:*": {
 			"env": [