CI: make testcafe tests stable (#28707)

.github/workflows/testcafe_tests.yml

@@ -124,8 +124,10 @@ jobs:
           { componentFolder: "scheduler/timezones", name: "scheduler (America/Los_Angeles)", timezone: "America/Los_Angeles" },
           { componentFolder: "form", name: "form (1/2)", indices: "1/2" },
           { componentFolder: "form", name: "form (2/2)", indices: "2/2" },
-          { componentFolder: "form", name: "form - material (1/2)", theme: 'material.blue.light', indices: "1/2" },
-          { componentFolder: "form", name: "form - material (2/2)", theme: 'material.blue.light', indices: "2/2" },
+          { componentFolder: "form", name: "form - material (1/4)", theme: 'material.blue.light', indices: "1/4" },
+          { componentFolder: "form", name: "form - material (2/4)", theme: 'material.blue.light', indices: "2/4" },
+          { componentFolder: "form", name: "form - material (3/4)", theme: 'material.blue.light', indices: "3/4" },
+          { componentFolder: "form", name: "form - material (4/4)", theme: 'material.blue.light', indices: "4/4" },
           { componentFolder: "form", name: "form - fluent (1/2)", theme: 'fluent.blue.light', indices: "1/2" },
           { componentFolder: "form", name: "form - fluent (2/2)", theme: 'fluent.blue.light', indices: "2/2" },
           { componentFolder: "editors", name: "editors (1/2)", indices: "1/2" },

apps/demos/testing/common.test.js

@@ -125,6 +125,9 @@ const SKIPPED_TESTS = {
     ],
     VectorMap: [
       { demo: 'Palette', themes: [THEME.material] },
+      { demo: 'Overview', themes: [THEME.material] },
+      { demo: 'PieMarkers', themes: [THEME.material] },
+      { demo: 'AreaWithLabelsAndTwoLegends', themes: [THEME.material] },
       { demo: 'CustomAnnotations', themes: [THEME.material] },
       { demo: 'CustomProjection', themes: [THEME.material] },
       { demo: 'MultipleLayers', themes: [THEME.material] },
@@ -136,9 +139,12 @@ const SKIPPED_TESTS = {
     Charts: [
       { demo: 'PiesWithEqualSize', themes: [THEME.material] },
       { demo: 'CustomAnnotations', themes: [THEME.material] },
+      { demo: 'ServerSideDataProcessing', themes: [THEME.material] },
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
     ],
     DataGrid: [
       { demo: 'SignalRService', themes: [THEME.material, THEME.fluent] },
+      { demo: 'EditStateManagement', themes: [THEME.material] },
       { demo: 'MultipleRecordSelectionModes', themes: [THEME.fluent] },
       { demo: 'ToolbarCustomization', themes: [THEME.fluent, THEME.material] },
       { demo: 'MultipleRecordSelectionAPI', themes: [THEME.material] },
@@ -156,6 +162,7 @@ const SKIPPED_TESTS = {
       { demo: 'ItemDragging', themes: [THEME.fluent, THEME.material] },
     ],
     VectorMap: [
+      { demo: 'Overview', themes: [THEME.material] },
       { demo: 'PieMarkers', themes: [THEME.material] },
       { demo: 'CustomAnnotations', themes: [THEME.material] },
       { demo: 'CustomMapData', themes: [THEME.material] },
@@ -174,6 +181,8 @@ const SKIPPED_TESTS = {
       { demo: 'LayoutCustomization', themes: [THEME.material] },
     ],
     Gauges: [
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
+      { demo: 'VariableNumberOfSubvalueIndicators', themes: [THEME.material] },
       { demo: 'DifferentValueIndicatorTypesLinearGauge', themes: [THEME.material] },
       { demo: 'ScaleLabelFormatting', themes: [THEME.material] },
     ],
@@ -183,9 +192,12 @@ const SKIPPED_TESTS = {
       { demo: 'TilingAlgorithms', themes: [THEME.material] },
       { demo: 'ExportAndPrintingAPI', themes: [THEME.material] },
       { demo: 'DiscreteAxisZoomingAndScrolling', themes: [THEME.material] },
+      { demo: 'Line', themes: [THEME.material] },
+      { demo: 'Spline', themes: [THEME.material] },
       { demo: 'Colorization', themes: [THEME.material] },
       { demo: 'SignalRService', themes: [THEME.material] },
       { demo: 'PointsAggregation', themes: [THEME.material] },
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
       { demo: 'AxisLabelsOverlapping', themes: [THEME.material] },
       { demo: 'ServerSideDataProcessing', themes: [THEME.material] },
       { demo: 'PiesWithEqualSize', themes: [THEME.material] },
@@ -196,6 +208,7 @@ const SKIPPED_TESTS = {
     ],
     DataGrid: [
       { demo: 'SignalRService', themes: [THEME.fluent, THEME.material] },
+      { demo: 'EditStateManagement', themes: [THEME.material] },
       { demo: 'ToolbarCustomization', themes: [THEME.fluent, THEME.material] },
       { demo: 'MultipleRecordSelectionModes', themes: [THEME.fluent] },
       { demo: 'FilteringAPI', themes: [THEME.material] },
@@ -228,7 +241,9 @@ const SKIPPED_TESTS = {
     ],
     Gauges: [
       { demo: 'Overview', themes: [THEME.material] },
+      { demo: 'VariableNumberOfSubvalueIndicators', themes: [THEME.material] },
       { demo: 'ScaleLabelFormatting', themes: [THEME.material] },
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
       { demo: 'DifferentValueIndicatorTypes', themes: [THEME.material] },
       { demo: 'DifferentValueIndicatorTypesLinearGauge', themes: [THEME.material] },
     ],
@@ -242,6 +257,9 @@ const SKIPPED_TESTS = {
     ],
     VectorMap: [
       { demo: 'CustomProjection', themes: [THEME.material] },
+      { demo: 'Spline', themes: [THEME.material] },
+      { demo: 'CustomAnnotations', themes: [THEME.material] },
+      { demo: 'Palette', themes: [THEME.material] },
       { demo: 'Overview', themes: [THEME.material] },
       { demo: 'PieMarkers', themes: [THEME.material] },
       { demo: 'TooltipHTMLSupport', themes: [THEME.material] },
@@ -250,6 +268,7 @@ const SKIPPED_TESTS = {
       { demo: 'AreaWithLabelsAndTwoLegends', themes: [THEME.material] },
     ],
     Gantt: [
+      { demo: 'ContextMenu', themes: [THEME.material] },
       { demo: 'Validation', themes: [THEME.generic, THEME.material, THEME.fluent] },
       { demo: 'TaskTemplate', themes: [THEME.generic, THEME.material, THEME.fluent] },
     ],

apps/demos/utils/visual-tests/matrix-test-helper.js

@@ -280,6 +280,7 @@ const SKIPPED_TESTS = {
       { demo: 'Overview', themes: [THEME.material] },
       { demo: 'ZoomingAndScrollingAPI', themes: [THEME.material] },
       { demo: 'ZoomingOnAreaSelection', themes: [THEME.material] },
+      { demo: 'CustomLegendMarkers', themes: [THEME.material] },
       { demo: 'DialogsAndNotificationsOverview', themes: [THEME.material] },
     ],
     VectorMap: [
@@ -307,6 +308,9 @@ const SKIPPED_TESTS = {
     Charts: [
       { demo: 'Overview', themes: [THEME.material] },
       { demo: 'ZoomingAndScrollingAPI', themes: [THEME.material] },
+      { demo: 'Crosshair', themes: [THEME.material] },
+      { demo: 'CustomAnnotations', themes: [THEME.material] },
+      { demo: 'CustomLegendMarkers', themes: [THEME.material] },
     ],
     DataGrid: [
       { demo: 'BatchEditing', themes: [THEME.fluent] },

e2e/testcafe-devextreme/helpers/themeUtils.ts

@@ -61,5 +61,7 @@ export async function testScreenshot(
       .ok();
   }
 
-  await changeTheme(process.env.theme ?? defaultThemeName);
+  if (isString(theme) || shouldTestInCompact) {
+    await changeTheme(process.env.theme ?? defaultThemeName);
+  }
 }

e2e/testcafe-devextreme/runner.js

@@ -59,7 +59,9 @@ createTestCafe({
             fs.rmSync('./screenshots', { recursive: true });
         }
 
-        const browsers = args.browsers.split(' ').map(expandBrowserAlias);
+        const browsers = args.browsers
+            .split(' ')
+            .map((browser) => expandBrowserAlias(browser, args.componentFolder.trim()));
         // eslint-disable-next-line no-console
         console.log('Browsers:', browsers);
 
@@ -113,24 +115,22 @@ createTestCafe({
             quarantineMode: { successThreshold: 1, attemptLimit: 5 },
         };
 
-        if(args.componentFolder.trim() !== 'renovation') {
-            runOptions.hooks = {
-                test: {
-                    before: async() => {
-                        if(args.shadowDom) {
-                            await addShadowRootTree();
-                        }
-
-                        if(args.theme) {
-                            await changeTheme(args.theme);
-                        }
-                    },
-                    after: async() => {
-                        await testPageUtils.clearTestPage();
+        runOptions.hooks = {
+            test: {
+                before: async() => {
+                    if(args.shadowDom) {
+                        await addShadowRootTree();
+                    }
+
+                    if(args.theme) {
+                        await changeTheme(args.theme);
                     }
                 },
-            };
-        }
+                after: async() => {
+                    await testPageUtils.clearTestPage();
+                }
+            },
+        };
 
         if(args.browsers === 'chrome:docker') {
             runOptions.disableScreenshots = true;
@@ -155,7 +155,7 @@ function setShadowDom(args) {
     process.env.shadowDom = args.shadowDom;
 }
 
-function expandBrowserAlias(browser) {
+function expandBrowserAlias(browser, componentFolder) {
     switch(browser) {
         case 'chrome:devextreme-shr2':
             return 'chrome:headless --disable-gpu --window-size=1200,800';

e2e/testcafe-devextreme/tests/chat/alertList.ts

@@ -75,10 +75,6 @@ test.clientScripts([
     timestamp: today,
     author: userFirst,
     text: 'Message text 4',
-  }, {
-    timestamp: today,
-    author: userFirst,
-    text: 'Message text 5',
   }];
 
   return createWidget('dxChat', {

e2e/testcafe-devextreme/tests/chat/messageGroup.ts

@@ -126,6 +126,9 @@ test('Messagegroup scenarios in disabled state', async (t) => {
       width: 250,
       height: 400,
     }, chatId);
+
+    const chat = new Chat(chatId);
+    await chat.repaint();
   });
 
   await testScreenshot(t, takeScreenshot, 'Messagegroup appearance in disabled state.png', { element: '#container' });

e2e/testcafe-devextreme/tests/chat/messageList.ts

@@ -9,6 +9,8 @@ import { createWidget } from '../../helpers/createWidget';
 import { testScreenshot } from '../../helpers/themeUtils';
 import { insertStylesheetRulesToPage } from '../../helpers/domUtils';
 
+const waitFont = ClientFunction(() => (window as any).DevExpress.ui.themes.waitWebFont('Item123somevalu*op ', 400));
+
 fixture.disablePageReloads`ChatMessageList`
   .page(url(__dirname, '../container.html'));
 
@@ -154,9 +156,7 @@ test('Messagelist should scrolled to the latest messages after being rendered in
 
 test('Messagelist with loadindicator appearance on initial loading', async (t) => {
   const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
-  const chat = new Chat('#container');
 
-  await chat.repaint();
   await testScreenshot(t, takeScreenshot, 'Messagelist loadindicator position on initial loading.png', { element: '#container' });
 
   await t
@@ -165,6 +165,8 @@ test('Messagelist with loadindicator appearance on initial loading', async (t) =
 }).before(async () => {
   await insertStylesheetRulesToPage('.dx-loadindicator-content, .dx-loadindicator-icon, .dx-loadindicator-segment, .dx-loadindicator-segment-inner { animation-play-state: paused !important; }');
 
+  await waitFont();
+
   await createWidget('dxChat', () => {
     const data: Message[] = [];
 

e2e/testcafe-devextreme/tests/chat/typingIndicator.ts

@@ -1,12 +1,14 @@
 import { createScreenshotsComparer } from 'devextreme-screenshot-comparer';
 import Chat from 'devextreme-testcafe-models/chat';
+import { ClientFunction } from 'testcafe';
 import { createUser, generateMessages } from './data';
 import url from '../../helpers/getPageUrl';
 import { createWidget } from '../../helpers/createWidget';
 import { getFullThemeName, testScreenshot } from '../../helpers/themeUtils';
 import { appendElementTo, insertStylesheetRulesToPage } from '../../helpers/domUtils';
 
 const CHAT_TYPINGINDICATOR_CIRCLE_CLASS = 'dx-chat-typingindicator-circle';
+const waitFont = ClientFunction(() => (window as any).DevExpress.ui.themes.waitWebFont('Item123somevalu*op ', 400));
 
 fixture.disablePageReloads`ChatTypingIndicator`
   .page(url(__dirname, '../container.html'));
@@ -15,8 +17,6 @@ test('Chat: typing indicator with emptyview', async (t) => {
   const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
   const chat = new Chat('#container');
 
-  await chat.repaint();
-
   await testScreenshot(t, takeScreenshot, 'Typing indicator with emptyview.png', {
     element: '#container',
     shouldTestInCompact: true,
@@ -44,6 +44,8 @@ test('Chat: typing indicator with emptyview', async (t) => {
     { name: 'Elodie Montclair' },
   ];
 
+  await waitFont();
+
   return createWidget('dxChat', {
     width: 400,
     height: 600,
@@ -53,6 +55,9 @@ test('Chat: typing indicator with emptyview', async (t) => {
 
 test('Chat: typing indicator with a lot of items', async (t) => {
   const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
+  const chat = new Chat('#chat');
+
+  await chat.repaint();
 
   await testScreenshot(t, takeScreenshot, 'Typing indicator with a lot of items.png', { element: '#chat' });
 
@@ -120,6 +125,8 @@ test('Chat: typing indicator', async (t) => {
 
   const typingUsers = [userFirst];
 
+  await waitFont();
+
   return createWidget('dxChat', {
     user: userSecond,
     width: 400,

e2e/testcafe-devextreme/tests/editors/dateBox/dateBoxGeometry.ts

@@ -15,9 +15,13 @@ safeSizeTest('Geometry is good', async (t) => {
   const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
   const dateBox = new DateBox('#container');
 
+  await dateBox.option('opened', true);
+
   await testScreenshot(t, takeScreenshot, 'Datebox with calendar.png');
 
+  await dateBox.option('opened', false);
   await dateBox.option('type', 'datetime');
+  await dateBox.option('opened', true);
 
   await testScreenshot(t, takeScreenshot, 'Datebox with datetime.png');
 
@@ -40,7 +44,6 @@ safeSizeTest('Geometry is good', async (t) => {
   await waitFont();
 
   return createWidget('dxDateBox', {
-    opened: true,
     pickerType: 'calendar',
     width: 200,
     value: new Date(1.5e12),