From 0212696e169439391fa9604175ea0865dd848cd9 Mon Sep 17 00:00:00 2001 From: Larry Wong Date: Thu, 14 Mar 2024 17:10:42 -0400 Subject: [PATCH] added playwright tests --- .github/workflows/test_perf.yml | 32 +- .../tests-examples/demo-todo-app.spec.js | 449 ------------------ playwright-tests/tests/simple.spec.js | 64 ++- tests/simple.spec.js | 31 -- 4 files changed, 70 insertions(+), 506 deletions(-) delete mode 100644 playwright-tests/tests-examples/demo-todo-app.spec.js delete mode 100644 tests/simple.spec.js diff --git a/.github/workflows/test_perf.yml b/.github/workflows/test_perf.yml index bd4205d862..f8d8b7b5eb 100644 --- a/.github/workflows/test_perf.yml +++ b/.github/workflows/test_perf.yml @@ -9,35 +9,43 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + with: + path: source - name: Set-up OCaml run: | sudo apt update -y sudo apt --assume-yes install curl m4 opam export OPAMYES=1 - opam init --compiler=ocaml-base-compiler.5.0.0 + opam init --compiler=ocaml-base-compiler.5.0.0 --disable-sandboxing - name: Install dependencies run: | eval $(opam env) export OPAMYES=1 make deps - - - name: Install system dependencies - run: make deps + working-directory: ./source - name: Install project development dependencies - run: make dev - - - name: Start web server in the background - run: make serve & - shell: bash + run: | + eval $(opam env) + make dev + working-directory: ./source - name: Setup Node.js uses: actions/setup-node@v3 - name: Install Playwright - run: npm install playwright + run: | + npm install + npx playwright install-deps + npx playwright install + working-directory: ./source/playwright-tests - - name: Run Playwright test script - run: npx playwright test tests/simple.spec.js \ No newline at end of file + - name: Start web server and run Playwright test script + run: | + eval $(opam env) + make serve > /dev/null & + cd playwright-tests + npx playwright test --reporter=list + working-directory: ./source \ No newline at end of file diff --git a/playwright-tests/tests-examples/demo-todo-app.spec.js b/playwright-tests/tests-examples/demo-todo-app.spec.js deleted file mode 100644 index e2eb87ce1b..0000000000 --- a/playwright-tests/tests-examples/demo-todo-app.spec.js +++ /dev/null @@ -1,449 +0,0 @@ -// @ts-check -const { test, expect } = require('@playwright/test'); - -test.beforeEach(async ({ page }) => { - await page.goto('https://demo.playwright.dev/todomvc'); -}); - -const TODO_ITEMS = [ - 'buy some cheese', - 'feed the cat', - 'book a doctors appointment' -]; - -test.describe('New Todo', () => { - test('should allow me to add todo items', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create 1st todo. - await newTodo.fill(TODO_ITEMS[0]); - await newTodo.press('Enter'); - - // Make sure the list only has one todo item. - await expect(page.getByTestId('todo-title')).toHaveText([ - TODO_ITEMS[0] - ]); - - // Create 2nd todo. - await newTodo.fill(TODO_ITEMS[1]); - await newTodo.press('Enter'); - - // Make sure the list now has two todo items. - await expect(page.getByTestId('todo-title')).toHaveText([ - TODO_ITEMS[0], - TODO_ITEMS[1] - ]); - - await checkNumberOfTodosInLocalStorage(page, 2); - }); - - test('should clear text input field when an item is added', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create one todo item. - await newTodo.fill(TODO_ITEMS[0]); - await newTodo.press('Enter'); - - // Check that input is empty. - await expect(newTodo).toBeEmpty(); - await checkNumberOfTodosInLocalStorage(page, 1); - }); - - test('should append new items to the bottom of the list', async ({ page }) => { - // Create 3 items. - await createDefaultTodos(page); - - // create a todo count locator - const todoCount = page.getByTestId('todo-count') - - // Check test using different methods. - await expect(page.getByText('3 items left')).toBeVisible(); - await expect(todoCount).toHaveText('3 items left'); - await expect(todoCount).toContainText('3'); - await expect(todoCount).toHaveText(/3/); - - // Check all items in one call. - await expect(page.getByTestId('todo-title')).toHaveText(TODO_ITEMS); - await checkNumberOfTodosInLocalStorage(page, 3); - }); -}); - -test.describe('Mark all as completed', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test.afterEach(async ({ page }) => { - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test('should allow me to mark all items as completed', async ({ page }) => { - // Complete all todos. - await page.getByLabel('Mark all as complete').check(); - - // Ensure all todos have 'completed' class. - await expect(page.getByTestId('todo-item')).toHaveClass(['completed', 'completed', 'completed']); - await checkNumberOfCompletedTodosInLocalStorage(page, 3); - }); - - test('should allow me to clear the complete state of all items', async ({ page }) => { - const toggleAll = page.getByLabel('Mark all as complete'); - // Check and then immediately uncheck. - await toggleAll.check(); - await toggleAll.uncheck(); - - // Should be no completed classes. - await expect(page.getByTestId('todo-item')).toHaveClass(['', '', '']); - }); - - test('complete all checkbox should update state when items are completed / cleared', async ({ page }) => { - const toggleAll = page.getByLabel('Mark all as complete'); - await toggleAll.check(); - await expect(toggleAll).toBeChecked(); - await checkNumberOfCompletedTodosInLocalStorage(page, 3); - - // Uncheck first todo. - const firstTodo = page.getByTestId('todo-item').nth(0); - await firstTodo.getByRole('checkbox').uncheck(); - - // Reuse toggleAll locator and make sure its not checked. - await expect(toggleAll).not.toBeChecked(); - - await firstTodo.getByRole('checkbox').check(); - await checkNumberOfCompletedTodosInLocalStorage(page, 3); - - // Assert the toggle all is checked again. - await expect(toggleAll).toBeChecked(); - }); -}); - -test.describe('Item', () => { - - test('should allow me to mark items as complete', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create two items. - for (const item of TODO_ITEMS.slice(0, 2)) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } - - // Check first item. - const firstTodo = page.getByTestId('todo-item').nth(0); - await firstTodo.getByRole('checkbox').check(); - await expect(firstTodo).toHaveClass('completed'); - - // Check second item. - const secondTodo = page.getByTestId('todo-item').nth(1); - await expect(secondTodo).not.toHaveClass('completed'); - await secondTodo.getByRole('checkbox').check(); - - // Assert completed class. - await expect(firstTodo).toHaveClass('completed'); - await expect(secondTodo).toHaveClass('completed'); - }); - - test('should allow me to un-mark items as complete', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create two items. - for (const item of TODO_ITEMS.slice(0, 2)) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } - - const firstTodo = page.getByTestId('todo-item').nth(0); - const secondTodo = page.getByTestId('todo-item').nth(1); - const firstTodoCheckbox = firstTodo.getByRole('checkbox'); - - await firstTodoCheckbox.check(); - await expect(firstTodo).toHaveClass('completed'); - await expect(secondTodo).not.toHaveClass('completed'); - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - - await firstTodoCheckbox.uncheck(); - await expect(firstTodo).not.toHaveClass('completed'); - await expect(secondTodo).not.toHaveClass('completed'); - await checkNumberOfCompletedTodosInLocalStorage(page, 0); - }); - - test('should allow me to edit an item', async ({ page }) => { - await createDefaultTodos(page); - - const todoItems = page.getByTestId('todo-item'); - const secondTodo = todoItems.nth(1); - await secondTodo.dblclick(); - await expect(secondTodo.getByRole('textbox', { name: 'Edit' })).toHaveValue(TODO_ITEMS[1]); - await secondTodo.getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); - await secondTodo.getByRole('textbox', { name: 'Edit' }).press('Enter'); - - // Explicitly assert the new text value. - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - 'buy some sausages', - TODO_ITEMS[2] - ]); - await checkTodosInLocalStorage(page, 'buy some sausages'); - }); -}); - -test.describe('Editing', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test('should hide other controls when editing', async ({ page }) => { - const todoItem = page.getByTestId('todo-item').nth(1); - await todoItem.dblclick(); - await expect(todoItem.getByRole('checkbox')).not.toBeVisible(); - await expect(todoItem.locator('label', { - hasText: TODO_ITEMS[1], - })).not.toBeVisible(); - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test('should save edits on blur', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).dispatchEvent('blur'); - - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - 'buy some sausages', - TODO_ITEMS[2], - ]); - await checkTodosInLocalStorage(page, 'buy some sausages'); - }); - - test('should trim entered text', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill(' buy some sausages '); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Enter'); - - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - 'buy some sausages', - TODO_ITEMS[2], - ]); - await checkTodosInLocalStorage(page, 'buy some sausages'); - }); - - test('should remove the item if an empty text string was entered', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill(''); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Enter'); - - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - TODO_ITEMS[2], - ]); - }); - - test('should cancel edits on escape', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Escape'); - await expect(todoItems).toHaveText(TODO_ITEMS); - }); -}); - -test.describe('Counter', () => { - test('should display the current number of todo items', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // create a todo count locator - const todoCount = page.getByTestId('todo-count') - - await newTodo.fill(TODO_ITEMS[0]); - await newTodo.press('Enter'); - await expect(todoCount).toContainText('1'); - - await newTodo.fill(TODO_ITEMS[1]); - await newTodo.press('Enter'); - await expect(todoCount).toContainText('2'); - - await checkNumberOfTodosInLocalStorage(page, 2); - }); -}); - -test.describe('Clear completed button', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - }); - - test('should display the correct text', async ({ page }) => { - await page.locator('.todo-list li .toggle').first().check(); - await expect(page.getByRole('button', { name: 'Clear completed' })).toBeVisible(); - }); - - test('should remove completed items when clicked', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).getByRole('checkbox').check(); - await page.getByRole('button', { name: 'Clear completed' }).click(); - await expect(todoItems).toHaveCount(2); - await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[2]]); - }); - - test('should be hidden when there are no items that are completed', async ({ page }) => { - await page.locator('.todo-list li .toggle').first().check(); - await page.getByRole('button', { name: 'Clear completed' }).click(); - await expect(page.getByRole('button', { name: 'Clear completed' })).toBeHidden(); - }); -}); - -test.describe('Persistence', () => { - test('should persist its data', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - for (const item of TODO_ITEMS.slice(0, 2)) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } - - const todoItems = page.getByTestId('todo-item'); - const firstTodoCheck = todoItems.nth(0).getByRole('checkbox'); - await firstTodoCheck.check(); - await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[1]]); - await expect(firstTodoCheck).toBeChecked(); - await expect(todoItems).toHaveClass(['completed', '']); - - // Ensure there is 1 completed item. - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - - // Now reload. - await page.reload(); - await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[1]]); - await expect(firstTodoCheck).toBeChecked(); - await expect(todoItems).toHaveClass(['completed', '']); - }); -}); - -test.describe('Routing', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - // make sure the app had a chance to save updated todos in storage - // before navigating to a new view, otherwise the items can get lost :( - // in some frameworks like Durandal - await checkTodosInLocalStorage(page, TODO_ITEMS[0]); - }); - - test('should allow me to display active items', async ({ page }) => { - const todoItem = page.getByTestId('todo-item'); - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - await page.getByRole('link', { name: 'Active' }).click(); - await expect(todoItem).toHaveCount(2); - await expect(todoItem).toHaveText([TODO_ITEMS[0], TODO_ITEMS[2]]); - }); - - test('should respect the back button', async ({ page }) => { - const todoItem = page.getByTestId('todo-item'); - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - - await test.step('Showing all items', async () => { - await page.getByRole('link', { name: 'All' }).click(); - await expect(todoItem).toHaveCount(3); - }); - - await test.step('Showing active items', async () => { - await page.getByRole('link', { name: 'Active' }).click(); - }); - - await test.step('Showing completed items', async () => { - await page.getByRole('link', { name: 'Completed' }).click(); - }); - - await expect(todoItem).toHaveCount(1); - await page.goBack(); - await expect(todoItem).toHaveCount(2); - await page.goBack(); - await expect(todoItem).toHaveCount(3); - }); - - test('should allow me to display completed items', async ({ page }) => { - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - await page.getByRole('link', { name: 'Completed' }).click(); - await expect(page.getByTestId('todo-item')).toHaveCount(1); - }); - - test('should allow me to display all items', async ({ page }) => { - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - await page.getByRole('link', { name: 'Active' }).click(); - await page.getByRole('link', { name: 'Completed' }).click(); - await page.getByRole('link', { name: 'All' }).click(); - await expect(page.getByTestId('todo-item')).toHaveCount(3); - }); - - test('should highlight the currently applied filter', async ({ page }) => { - await expect(page.getByRole('link', { name: 'All' })).toHaveClass('selected'); - - //create locators for active and completed links - const activeLink = page.getByRole('link', { name: 'Active' }); - const completedLink = page.getByRole('link', { name: 'Completed' }); - await activeLink.click(); - - // Page change - active items. - await expect(activeLink).toHaveClass('selected'); - await completedLink.click(); - - // Page change - completed items. - await expect(completedLink).toHaveClass('selected'); - }); -}); - -async function createDefaultTodos(page) { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - for (const item of TODO_ITEMS) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } -} - -/** - * @param {import('@playwright/test').Page} page - * @param {number} expected - */ - async function checkNumberOfTodosInLocalStorage(page, expected) { - return await page.waitForFunction(e => { - return JSON.parse(localStorage['react-todos']).length === e; - }, expected); -} - -/** - * @param {import('@playwright/test').Page} page - * @param {number} expected - */ - async function checkNumberOfCompletedTodosInLocalStorage(page, expected) { - return await page.waitForFunction(e => { - return JSON.parse(localStorage['react-todos']).filter(i => i.completed).length === e; - }, expected); -} - -/** - * @param {import('@playwright/test').Page} page - * @param {string} title - */ -async function checkTodosInLocalStorage(page, title) { - return await page.waitForFunction(t => { - return JSON.parse(localStorage['react-todos']).map(i => i.title).includes(t); - }, title); -} diff --git a/playwright-tests/tests/simple.spec.js b/playwright-tests/tests/simple.spec.js index 59f42c042b..10cb10f765 100644 --- a/playwright-tests/tests/simple.spec.js +++ b/playwright-tests/tests/simple.spec.js @@ -1,6 +1,35 @@ -const { test, expect } = require('@playwright/test'); +import { test, expect } from '@playwright/test'; -test('measure time from key press to page finish changing', async ({ page }) => { +const program1 = `1+1`; +const program2 = `let double_recursively : Int -> Int = +fun n -> +if n == 0 then 0 +else double_recursively(n - 1) + 2 +in +let empty_list : [Int] = [] in +let non_empty_list : [Int] = 1::2::3::[] in +let list_literals : [Int] = [1, 2, 3] in +let length : [Int] -> Int = +fun xs -> +case xs +| [] => 0 +| hd::tl => 1 + length(tl) +end +in +let has_at_least_two_elements : [Int] -> Bool = +fun xs -> +case xs +| [] => false +| hd::[] => false +| a::b::[] => true +end +in +let string_lits = "Hello, world!" in +let string_equality = string_lits $== "Hello, world!" in +let non_empty_hole : Int = true in +2 + 2`; + +const testTemplate = async (page, program, expectedTime) => { await page.goto('http://0.0.0.0:8000', { waitUntil: 'domcontentloaded' }); // The selector for a div with the ID "page" @@ -12,20 +41,27 @@ test('measure time from key press to page finish changing', async ({ page }) => // Start the timer by injecting performance.now() before the last key press const startTime = await page.evaluate(() => performance.now()); - // Press the keys '1', '+', '1' individually - await page.keyboard.press('1'); - await page.keyboard.press('NumpadAdd'); // For the '+' key on the numpad - await page.keyboard.press('1'); - - // Selector that matches the div with class "ok result" - const resultSelector = 'div.ok.result'; - await page.waitForSelector(resultSelector); + const resultSelector = 'div.ok.status'; + for (let c of program) { + await page.keyboard.press(c); + // Selector that matches the div with class "ok result" + await page.waitForSelector(resultSelector); + } // Stop the timer const endTime = await page.evaluate(() => performance.now()); - const duration = endTime - startTime; - console.log(`Time elapsed: ${duration} milliseconds`); + const elapsedTime = endTime - startTime; + console.log(`Time elapsed: ${elapsedTime} milliseconds`); + + expect(elapsedTime).toBeLessThan(expectedTime); +}; + +test('measure program 1', async ({ page }) => { + test.setTimeout(10000); // 10 sec timeout + await testTemplate(page, program1, 5000); +}); - // Verify the duration is within an expected range: 5s - expect(duration).toBeLessThan(5000); +test('measure program 2', async ({ page }) => { + test.setTimeout(120000); // 120 sec timeout + await testTemplate(page, program2, 120000); }); \ No newline at end of file diff --git a/tests/simple.spec.js b/tests/simple.spec.js deleted file mode 100644 index 59f42c042b..0000000000 --- a/tests/simple.spec.js +++ /dev/null @@ -1,31 +0,0 @@ -const { test, expect } = require('@playwright/test'); - -test('measure time from key press to page finish changing', async ({ page }) => { - await page.goto('http://0.0.0.0:8000', { waitUntil: 'domcontentloaded' }); - - // The selector for a div with the ID "page" - const pageDivSelector = '#page'; - // Wait for the div to be present in the DOM and to be visible. - await page.waitForSelector(pageDivSelector, { state: 'visible' }); - await page.click(pageDivSelector); - - // Start the timer by injecting performance.now() before the last key press - const startTime = await page.evaluate(() => performance.now()); - - // Press the keys '1', '+', '1' individually - await page.keyboard.press('1'); - await page.keyboard.press('NumpadAdd'); // For the '+' key on the numpad - await page.keyboard.press('1'); - - // Selector that matches the div with class "ok result" - const resultSelector = 'div.ok.result'; - await page.waitForSelector(resultSelector); - - // Stop the timer - const endTime = await page.evaluate(() => performance.now()); - const duration = endTime - startTime; - console.log(`Time elapsed: ${duration} milliseconds`); - - // Verify the duration is within an expected range: 5s - expect(duration).toBeLessThan(5000); -}); \ No newline at end of file