Add visual regression tests

.gitignore

@@ -5,3 +5,8 @@ backend/mirror.gif
 backend/mirror.png
 backend/mirror.jpg
 backend/GeoIP.dat
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

.htaccess

@@ -1,4 +1,4 @@
-<FilesMatch "\.(inc|sql)$">
+<FilesMatch "\.(inc|sql|lock)$">
   Order allow,deny
   Deny from all
 </FilesMatch>

Makefile

@@ -21,7 +21,20 @@ help: ## Displays this list of targets with descriptions
 tests: vendor ## Runs unit and end-to-end tests with phpunit/phpunit
 	vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=unit
 	rm -rf tests/server.log
-	tests/server start; vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end; tests/server stop
+	tests/server start;
+	vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end;
+	npx playwright test
+	tests/server stop
+
+tests_visual:
+	tests/server start;
+	npx playwright test
+	tests/server stop
+
+tests_update_snapshots:
+	tests/server start;
+	npx playwright test --update-snapshots
+	tests/server stop
 
 vendor: composer.json composer.lock
 	composer validate --strict

package.json

@@ -0,0 +1,8 @@
+{
+  "name": "php.net",
+  "devDependencies": {
+    "@playwright/test": "^1.44.0",
+    "@types/node": "^20.12.11"
+  },
+  "scripts": {}
+}

playwright.config.ts

@@ -0,0 +1,34 @@
+import {defineConfig, devices} from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+    testDir: './tests/Visual',
+    /* Run tests in files in parallel */
+    fullyParallel: true,
+    /* Fail the build on CI if you accidentally left test.only in the source code. */
+    forbidOnly: !!process.env.CI,
+    /* Retry on CI only */
+    retries: process.env.CI ? 2 : 0,
+    /* Opt out of parallel tests on CI. */
+    workers: process.env.CI ? 1 : undefined,
+    /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+    reporter: 'html',
+    /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+    use: {
+        /* Base URL to use in actions like `await page.goto('/')`. */
+        // baseURL: 'http://127.0.0.1:3000',
+
+        /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+        trace: 'on-first-retry',
+    },
+    timeout: 0,
+
+    projects: [
+        {
+            name: 'chromium',
+            use: {...devices['Desktop Chrome']},
+        },
+    ],
+});

tests/Visual/SmokeTest.spec.ts

@@ -0,0 +1,61 @@
+import {expect, test} from '@playwright/test';
+
+export type TestPageOptions = {
+    path: string
+    options?: object
+    evaluate?: () => any
+}
+
+const items: TestPageOptions[] = [
+    {
+        path: 'index.php',
+        evaluate: () => {
+            const selector = document.querySelector('.elephpants');
+            selector.remove()
+        }
+    },
+    {
+        path: 'archive/1998.php',
+        evaluate: () => {
+            const selector = document.querySelector('.elephpants');
+            selector.remove()
+        }
+    },
+    {path: 'releases/8_3_6.php'},
+    {path: 'releases/8.0/index.php'},
+    {path: 'releases/8.1/index.php'},
+    {path: 'releases/8.2/index.php'},
+    {path: 'releases/8.3/index.php'},
+    {path: 'manual/index.php'},
+    {path: 'manual/php5.php'},
+    {
+        path: 'conferences/index.php',
+        options: {
+            fullPage: false,
+        }
+    },
+]
+
+for (const item of items) {
+    test(`testing with ${item.path}`, async ({page}) => {
+        const httpHost = process.env.HTTP_HOST
+
+        if (typeof httpHost !== 'string') {
+            throw new Error('Environment variable "HTTP_HOST" is not set.')
+        }
+
+        await page.goto(`${httpHost}/${item.path}`)
+
+        if (typeof item.evaluate === 'function') {
+            await page.evaluate(item.evaluate)
+        }
+
+        await expect(page).toHaveScreenshot(
+            `tests/screenshots/${item.path}.png`,
+            item.options ?? {
+                fullPage: true,
+                timeout: 10000,
+            }
+        )
+    })
+}

yarn.lock

@@ -0,0 +1,41 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@playwright/test@^1.44.0":
+  version "1.44.0"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.44.0.tgz#ac7a764b5ee6a80558bdc0fcbc525fcb81f83465"
+  integrity sha512-rNX5lbNidamSUorBhB4XZ9SQTjAqfe5M+p37Z8ic0jPFBMo5iCtQz1kRWkEMg+rYOKSlVycpQmpqjSFq7LXOfg==
+  dependencies:
+    playwright "1.44.0"
+
+"@types/node@^20.12.11":
+  version "20.12.11"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.11.tgz#c4ef00d3507000d17690643278a60dc55a9dc9be"
+  integrity sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==
+  dependencies:
+    undici-types "~5.26.4"
+
+[email protected]:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
+[email protected]:
+  version "1.44.0"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.44.0.tgz#316c4f0bca0551ffb88b6eb1c97bc0d2d861b0d5"
+  integrity sha512-ZTbkNpFfYcGWohvTTl+xewITm7EOuqIqex0c7dNZ+aXsbrLj0qI8XlGKfPpipjm0Wny/4Lt4CJsWJk1stVS5qQ==
+
+[email protected]:
+  version "1.44.0"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.44.0.tgz#22894e9b69087f6beb639249323d80fe2b5087ff"
+  integrity sha512-F9b3GUCLQ3Nffrfb6dunPOkE5Mh68tR7zN32L4jCk4FjQamgesGay7/dAAe1WaMEGV04DkdJfcJzjoCKygUaRQ==
+  dependencies:
+    playwright-core "1.44.0"
+  optionalDependencies:
+    fsevents "2.3.2"
+
+undici-types@~5.26.4:
+  version "5.26.5"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==