Skip to content

Commit

Permalink
Adjust tests for electron playwright
Browse files Browse the repository at this point in the history
* Update theia dependency to 1.46.1
* Use new TheiaApploader
* Add workaround for Electron playwright multi root issue
* Support different projects (e.g., chromium or Electron) in performance reports

Contributed on behalf of STMicroelectronics

Signed-off-by: Olaf Lessenich <[email protected]>
  • Loading branch information
xai committed Mar 26, 2024
1 parent 97ecba0 commit 5c31aa0
Show file tree
Hide file tree
Showing 10 changed files with 316 additions and 88 deletions.
94 changes: 92 additions & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ on:
jobs:
build:
name: E2E Tests on ubuntu-latest with Node.js 16.x

runs-on: ubuntu-latest
timeout-minutes: 60

Expand Down Expand Up @@ -50,20 +49,104 @@ jobs:
run: |
yarn --skip-integrity-check --network-timeout 100000
yarn browser build
yarn electron build
env:
NODE_OPTIONS: --max_old_space_size=4096
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9

- name: Cache Build Result
uses: actions/cache@v3
id: build-result
with:
path: ./*
key: ${{ github.run_number }}

browser-tests:
name: E2E Browser Tests on ubuntu-latest with Node.js 16.x
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [build]

steps:
- name: Restore Build Result (#${{ github.run_number }})
uses: actions/cache/restore@v3
id: build-result
with:
path: ./*
key: ${{ github.run_number }}
fail-on-cache-miss: true

- name: Use Node.js "16.x"
uses: actions/setup-node@v3
with:
node-version: "16.x"
registry-url: "https://registry.npmjs.org"

- name: Use Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"

- name: Run Theia
shell: bash
working-directory: ./theia
run: yarn browser start &

- name: Test (playwright)
- name: Test (playwright browser)
uses: GabrielBB/xvfb-action@v1
with:
run: yarn ui-tests-ci

- name: Upload test results (browser)
uses: actions/upload-artifact@v3
with:
name: ui-tests-browser-${{ github.run_number }}
path: allure_results/*

electron-tests:
name: E2E Electron Tests on ubuntu-latest with Node.js 16.x
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [build]

steps:
- name: Restore Build Result (#${{ github.run_number }})
uses: actions/cache/restore@v3
id: build-result
with:
path: ./*
key: ${{ github.run_number }}
fail-on-cache-miss: true

- name: Use Node.js "16.x"
uses: actions/setup-node@v3
with:
node-version: "16.x"
registry-url: "https://registry.npmjs.org"

- name: Use Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"

- name: Test (playwright electron)
uses: GabrielBB/xvfb-action@v1
with:
run: yarn ui-tests-ci-electron

- name: Upload test results (electron)
uses: actions/upload-artifact@v3
with:
name: ui-tests-electron-${{ github.run_number }}
path: allure_results/*

generate-report:
name: E2E Electron Tests on ubuntu-latest with Node.js 16.x
runs-on: ubuntu-latest
needs: [browser-tests, electron-tests]

steps:

- name: Get History
uses: actions/checkout@v2
if: always() && github.ref == 'refs/heads/main'
Expand All @@ -72,6 +155,13 @@ jobs:
ref: gh-pages
path: gh-pages

- name: Download test results
uses: actions/download-artifact@v3
with:
path: ui-tests-*-${{ github.run_number }}
merge-multiple: true
path: allure_results

- name: Generate Report
uses: simple-elf/allure-report-action@master
if: always() && github.ref == 'refs/heads/main'
Expand Down
75 changes: 65 additions & 10 deletions .github/workflows/performance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ jobs:
shell: bash
run: |
cp -f ./configs/theia.production.webpack.config.js ./theia/examples/browser/webpack.config.js
cp -f ./configs/theia.production.webpack.config.js ./theia/examples/electron/webpack.config.js
- name: Build Theia
shell: bash
working-directory: ./theia
Expand All @@ -51,15 +52,23 @@ jobs:
env:
NODE_OPTIONS: --max_old_space_size=4096
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
- name: Build Electron App (Production)
shell: bash
working-directory: ./theia/examples/electron
run: |
yarn -s compile && yarn rebuild && yarn theia build --mode=production
env:
NODE_OPTIONS: --max_old_space_size=4096
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
- name: Cache Build Result
uses: actions/cache@v3
id: build-result
with:
path: ./*
key: ${{ github.run_number }}

run-test:
name: Performance benchmark on ubuntu-latest with Node.js 16.x
run-browser-tests:
name: Performance browser benchmark on ubuntu-latest with Node.js 16.x
runs-on: ubuntu-latest
needs: [prepare]
strategy:
Expand Down Expand Up @@ -88,26 +97,67 @@ jobs:
with:
ref: gh-pages
path: gh-pages
- name: Run Theia (#${{ matrix.run }})
- name: Run Theia browser backend (#${{ matrix.run }})
shell: bash
working-directory: ./theia
run: yarn browser start:debug &
- name: Run Performance Measurement (#${{ matrix.run }})
uses: GabrielBB/xvfb-action@v1
- name: Run Performance Measurement: browser (#${{ matrix.run }})
env:
RUN_NO: ${{ matrix.run }}
with:
run: yarn performance
- name: Upload performance measurement (#${{ matrix.run }})
- name: Upload performance measurement: browser (#${{ matrix.run }})
uses: actions/upload-artifact@v3
with:
name: performance-measurement-browser-${{ github.run_number }}
path: performance-metrics/*

run-electron-tests:
name: Performance electron benchmark on ubuntu-latest with Node.js 16.x
runs-on: ubuntu-latest
needs: [prepare]
strategy:
matrix:
run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
timeout-minutes: 40
steps:
- name: Restore Build Result (#${{ matrix.run }})
uses: actions/cache/restore@v3
id: build-result
with:
path: ./*
key: ${{ github.run_number }}
fail-on-cache-miss: true
- name: Use Node.js "16.x"
uses: actions/setup-node@v3
with:
node-version: "16.x"
registry-url: "https://registry.npmjs.org"
- name: Use Python 3.x
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Get History
uses: actions/checkout@v4
with:
ref: gh-pages
path: gh-pages
- name: Run Performance Measurement: electron (#${{ matrix.run }})
uses: GabrielBB/xvfb-action@v1
env:
RUN_NO: ${{ matrix.run }}
with:
run: yarn performance-on-electron
- name: Upload performance measurement: electron (#${{ matrix.run }})
uses: actions/upload-artifact@v3
with:
name: performance-measurement-${{ github.run_number }}
name: performance-measurement-electron-${{ github.run_number }}
path: performance-metrics/*

generate-report:
name: Performance benchmark on ubuntu-latest with Node.js 16.x
runs-on: ubuntu-latest
needs: [run-test]
needs: [run-browser-tests, run-electron-tests]
steps:
- name: Restore Build Result
uses: actions/cache/restore@v3
Expand All @@ -126,12 +176,17 @@ jobs:
- name: Download Performance Measurements
uses: actions/download-artifact@v3
with:
name: performance-measurement-${{ github.run_number }}
pattern: performance-measurement-*-${{ github.run_number }}
merge-multiple: true
path: performance-metrics
- name: Prepare Report
- name: Prepare Browser Report
if: always() && github.ref == 'refs/heads/main'
shell: bash
run: yarn performance-report
- name: Prepare Electron Report
if: always() && github.ref == 'refs/heads/main'
shell: bash
run: yarn performance-report-electron
- name: Publish Report
if: always() && github.ref == 'refs/heads/main'
uses: peaceiris/actions-gh-pages@v2
Expand Down
6 changes: 4 additions & 2 deletions configs/performance.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import { MetricsFetchConfig } from '../scripts/fetch-metrics';
export default defineConfig({
testDir: '../lib/tests',
testMatch: ['**/*.performance.js'],
globalTeardown: require.resolve('../scripts/fetch-metrics.ts'),
workers: 1,
timeout: 60 * 1000,
use: {
Expand Down Expand Up @@ -51,6 +50,9 @@ export default defineConfig({
...devices['Desktop Chrome'],
channel: 'chrome'
},
}
},
{
name: 'Electron',
},
]
});
8 changes: 8 additions & 0 deletions configs/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ const config: PlaywrightTestConfig = {
reporter: [
['list'],
['allure-playwright']
],
projects: [
{
name: 'Browser',
},
{
name: 'Electron',
}
]
};

Expand Down
34 changes: 19 additions & 15 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,37 +1,41 @@
{
"private": true,
"name": "theia-e2e-test-suite",
"version": "1.39.0",
"version": "1.46.1",
"license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
"scripts": {
"prepare": "yarn clean && yarn build",
"clean": "rimraf lib tsconfig.tsbuildinfo",
"build": "tsc --incremental && if-env SKIP_CHROMIUM_INSTALLATION=1 && echo 'Skipping chromium install' || npx playwright install chromium",
"ui-tests": "yarn && playwright test --config=./configs/playwright.config.ts",
"ui-tests-ci": "yarn && playwright test --config=./configs/playwright.ci.config.ts",
"ui-tests-headful": "yarn && playwright test --config=./configs/playwright.headful.config.ts",
"ui-tests": "yarn && playwright test --config=./configs/playwright.config.ts --project=Browser",
"ui-tests-electron": "yarn && USE_ELECTRON=true playwright test --config=./configs/playwright.config.ts --project=Electron",
"ui-tests-ci": "yarn && playwright test --config=./configs/playwright.ci.config.ts --project=Browser",
"ui-tests-ci-electron": "yarn && USE_ELECTRON=true playwright test --config=./configs/playwright.ci.config.ts --project=Electron",
"ui-tests-headful": "yarn && playwright test --config=./configs/playwright.headful.config.ts --project=Browser",
"ui-tests-report-generate": "allure generate ./allure-results --clean -o allure-results/allure-report",
"ui-tests-report": "yarn ui-tests-report-generate && allure open allure-results/allure-report",
"performance": "yarn run performance-on-chromium",
"performance-on-chromium": "yarn && playwright test --config=./configs/performance.config.ts --project=chromium",
"performance-on-chrome": "yarn && playwright test --config=./configs/performance.config.ts --project=Chrome",
"performance-report": "yarn && node lib/scripts/performance-report.js"
"performance-on-electron": "yarn && USE_ELECTRON=true playwright test --config=./configs/performance.config.ts --project=Electron",
"performance-report": "yarn && node lib/scripts/performance-report.js --performanceMetricsPath performance-metrics/chromium --performancePublishPath=performance/chromium",
"performance-report-electron": "yarn && node lib/scripts/performance-report.js --performanceMetricsPath performance-metrics/Electron --performancePublishPath=performance/Electron"
},
"dependencies": {
"@playwright/test": "^1.35.1",
"@theia/playwright": "^1.39.0"
"@playwright/test": "^1.37.1",
"@theia/playwright": "^1.46.1"
},
"devDependencies": {
"allure-commandline": "^2.23.0",
"allure-playwright": "^2.4.0",
"rimraf": "^3.0.0",
"node-fetch": "^2.6.7",
"@types/node-fetch": "^2.6.4",
"fs-extra": "^11.1.1",
"@types/fs-extra": "^11.0.1",
"yargs": "^17.7.2",
"@types/node-fetch": "^2.6.4",
"@types/yargs": "^17.0.24",
"allure-commandline": "^2.23.1",
"allure-playwright": "^2.5.0",
"fs-extra": "^11.1.1",
"if-env": "^1.0.4",
"typescript": "~4.5.5"
"node-fetch": "^2.6.7",
"rimraf": "^3.0.0",
"typescript": "~4.5.5",
"yargs": "^17.7.2"
}
}
18 changes: 11 additions & 7 deletions scripts/fetch-metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ import { type FullConfig } from '@playwright/test';
import * as fs from 'fs-extra';
import fetch from 'node-fetch';

async function fetchMetrics(config: FullConfig) {
await fetchPerformanceMetrics(config.metadata.performanceMetrics, config.metadata.totalTime);
async function fetchMetrics(projectName: string, config: FullConfig) {
await fetchPerformanceMetrics(projectName, config.metadata.performanceMetrics, config.metadata.totalTime);
}

export default fetchMetrics;
Expand All @@ -31,13 +31,17 @@ export interface MetricsFetchConfig {
outputFilePath: string;
}

export async function fetchPerformanceMetrics({
metricsEndpoint,
outputFileNamePrefix,
outputFileNamePostfix,
outputFilePath }: MetricsFetchConfig,
export async function fetchPerformanceMetrics(
projectName: string,
{
metricsEndpoint,
outputFileNamePrefix,
outputFileNamePostfix,
outputFilePath
}: MetricsFetchConfig,
totalTime?: string
) {
outputFilePath = `${outputFilePath}/${projectName}`;
const now = new Date();
const dateString = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`;
const timeString = `${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`;
Expand Down
Loading

0 comments on commit 5c31aa0

Please sign in to comment.