CI: make testcafe tests stable (DevExpress#28707)

EugeniyKiyashko · Jan 15, 2025 · 8aaa01e · 8aaa01e
1 parent abdd30e
commit 8aaa01e
Show file tree

Hide file tree

Showing 51 changed files with 402 additions and 331 deletions.
diff --git a/.github/workflows/testcafe_tests.yml b/.github/workflows/testcafe_tests.yml
@@ -101,8 +101,10 @@ jobs:
           { componentFolder: "scheduler/timezones", name: "scheduler (America/Los_Angeles)", timezone: "America/Los_Angeles" },
           { componentFolder: "form", name: "form (1/2)", indices: "1/2" },
           { componentFolder: "form", name: "form (2/2)", indices: "2/2" },
-          { componentFolder: "form", name: "form - material (1/2)", theme: 'material.blue.light', indices: "1/2" },
-          { componentFolder: "form", name: "form - material (2/2)", theme: 'material.blue.light', indices: "2/2" },
+          { componentFolder: "form", name: "form - material (1/4)", theme: 'material.blue.light', indices: "1/4" },
+          { componentFolder: "form", name: "form - material (2/4)", theme: 'material.blue.light', indices: "2/4" },
+          { componentFolder: "form", name: "form - material (3/4)", theme: 'material.blue.light', indices: "3/4" },
+          { componentFolder: "form", name: "form - material (4/4)", theme: 'material.blue.light', indices: "4/4" },
           { componentFolder: "form", name: "form - fluent (1/2)", theme: 'fluent.blue.light', indices: "1/2" },
           { componentFolder: "form", name: "form - fluent (2/2)", theme: 'fluent.blue.light', indices: "2/2" },
           { componentFolder: "editors", name: "editors (1/2)", indices: "1/2" },

diff --git a/apps/demos/testing/common.test.js b/apps/demos/testing/common.test.js
@@ -120,6 +120,9 @@ const SKIPPED_TESTS = {
     ],
     VectorMap: [
       { demo: 'Palette', themes: [THEME.material] },
+      { demo: 'Overview', themes: [THEME.material] },
+      { demo: 'PieMarkers', themes: [THEME.material] },
+      { demo: 'AreaWithLabelsAndTwoLegends', themes: [THEME.material] },
       { demo: 'CustomAnnotations', themes: [THEME.material] },
       { demo: 'CustomProjection', themes: [THEME.material] },
       { demo: 'MultipleLayers', themes: [THEME.material] },
@@ -131,9 +134,12 @@ const SKIPPED_TESTS = {
     Charts: [
       { demo: 'PiesWithEqualSize', themes: [THEME.material] },
       { demo: 'CustomAnnotations', themes: [THEME.material] },
+      { demo: 'ServerSideDataProcessing', themes: [THEME.material] },
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
     ],
     DataGrid: [
       { demo: 'SignalRService', themes: [THEME.material, THEME.fluent] },
+      { demo: 'EditStateManagement', themes: [THEME.material] },
       { demo: 'MultipleRecordSelectionModes', themes: [THEME.fluent] },
       { demo: 'ToolbarCustomization', themes: [THEME.fluent, THEME.material] },
       { demo: 'MultipleRecordSelectionAPI', themes: [THEME.material] },
@@ -148,6 +154,7 @@ const SKIPPED_TESTS = {
       { demo: 'ItemDragging', themes: [THEME.fluent, THEME.material] },
     ],
     VectorMap: [
+      { demo: 'Overview', themes: [THEME.material] },
       { demo: 'PieMarkers', themes: [THEME.material] },
       { demo: 'CustomMapData', themes: [THEME.material] },
       { demo: 'CustomProjection', themes: [THEME.material] },
@@ -164,6 +171,8 @@ const SKIPPED_TESTS = {
       { demo: 'LayoutCustomization', themes: [THEME.material] },
     ],
     Gauges: [
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
+      { demo: 'VariableNumberOfSubvalueIndicators', themes: [THEME.material] },
       { demo: 'DifferentValueIndicatorTypesLinearGauge', themes: [THEME.material] },
       { demo: 'ScaleLabelFormatting', themes: [THEME.material] },
     ],
@@ -173,9 +182,12 @@ const SKIPPED_TESTS = {
       { demo: 'TilingAlgorithms', themes: [THEME.material] },
       { demo: 'ExportAndPrintingAPI', themes: [THEME.material] },
       { demo: 'DiscreteAxisZoomingAndScrolling', themes: [THEME.material] },
+      { demo: 'Line', themes: [THEME.material] },
+      { demo: 'Spline', themes: [THEME.material] },
       { demo: 'Colorization', themes: [THEME.material] },
       { demo: 'SignalRService', themes: [THEME.material] },
       { demo: 'PointsAggregation', themes: [THEME.material] },
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
       { demo: 'AxisLabelsOverlapping', themes: [THEME.material] },
       { demo: 'ServerSideDataProcessing', themes: [THEME.material] },
       { demo: 'PiesWithEqualSize', themes: [THEME.material] },
@@ -186,6 +198,7 @@ const SKIPPED_TESTS = {
     ],
     DataGrid: [
       { demo: 'SignalRService', themes: [THEME.fluent, THEME.material] },
+      { demo: 'EditStateManagement', themes: [THEME.material] },
       { demo: 'ToolbarCustomization', themes: [THEME.fluent, THEME.material] },
       { demo: 'MultipleRecordSelectionModes', themes: [THEME.fluent] },
       { demo: 'FilteringAPI', themes: [THEME.material] },
@@ -202,7 +215,9 @@ const SKIPPED_TESTS = {
     ],
     Gauges: [
       { demo: 'Overview', themes: [THEME.material] },
+      { demo: 'VariableNumberOfSubvalueIndicators', themes: [THEME.material] },
       { demo: 'ScaleLabelFormatting', themes: [THEME.material] },
+      { demo: 'SubvalueIndicatorTextFormatting', themes: [THEME.material] },
       { demo: 'DifferentValueIndicatorTypes', themes: [THEME.material] },
       { demo: 'DifferentValueIndicatorTypesLinearGauge', themes: [THEME.material] },
     ],
@@ -215,6 +230,10 @@ const SKIPPED_TESTS = {
       { demo: 'LayoutCustomization', themes: [THEME.material] },
     ],
     VectorMap: [
+      { demo: 'CustomProjection', themes: [THEME.material] },
+      { demo: 'Spline', themes: [THEME.material] },
+      { demo: 'CustomAnnotations', themes: [THEME.material] },
+      { demo: 'Palette', themes: [THEME.material] },
       { demo: 'Overview', themes: [THEME.material] },
       { demo: 'PieMarkers', themes: [THEME.material] },
       { demo: 'TooltipHTMLSupport', themes: [THEME.material] },
@@ -223,7 +242,9 @@ const SKIPPED_TESTS = {
       { demo: 'AreaWithLabelsAndTwoLegends', themes: [THEME.material] },
     ],
     Gantt: [
-      { demo: 'Validation', themes: [THEME.material] },
+      { demo: 'ContextMenu', themes: [THEME.material] },
+      { demo: 'Validation', themes: [THEME.generic, THEME.material, THEME.fluent] },
+      { demo: 'TaskTemplate', themes: [THEME.generic, THEME.material, THEME.fluent] },
     ],
   },
 };

diff --git a/...widgets/datagrid/etalons/datagrid_filtering_2_desktop (material.blue.light).png b/...widgets/datagrid/etalons/datagrid_filtering_2_desktop (material.blue.light).png
diff --git a/...widgets/datagrid/etalons/datagrid_filtering_3_desktop (material.blue.light).png b/...widgets/datagrid/etalons/datagrid_filtering_3_desktop (material.blue.light).png
diff --git a/...widgets/datagrid/etalons/datagrid_filtering_4_desktop (material.blue.light).png b/...widgets/datagrid/etalons/datagrid_filtering_4_desktop (material.blue.light).png
diff --git a/...widgets/datagrid/etalons/datagrid_filtering_5_desktop (material.blue.light).png b/...widgets/datagrid/etalons/datagrid_filtering_5_desktop (material.blue.light).png
diff --git a/...widgets/datagrid/etalons/datagrid_filtering_6_desktop (material.blue.light).png b/...widgets/datagrid/etalons/datagrid_filtering_6_desktop (material.blue.light).png
diff --git a/...dgets/datagrid/etalons/datagrid_filtering_7_1_desktop (material.blue.light).png b/...dgets/datagrid/etalons/datagrid_filtering_7_1_desktop (material.blue.light).png
diff --git a/...widgets/datagrid/etalons/datagrid_filtering_7_desktop (material.blue.light).png b/...widgets/datagrid/etalons/datagrid_filtering_7_desktop (material.blue.light).png
diff --git a/...widgets/datagrid/etalons/datagrid_filtering_8_desktop (material.blue.light).png b/...widgets/datagrid/etalons/datagrid_filtering_8_desktop (material.blue.light).png
diff --git a/...atagrid/etalons/datagrid_filtering_by_range_1_desktop (material.blue.light).png b/...atagrid/etalons/datagrid_filtering_by_range_1_desktop (material.blue.light).png
diff --git a/...atagrid/etalons/datagrid_filtering_by_range_2_desktop (material.blue.light).png b/...atagrid/etalons/datagrid_filtering_by_range_2_desktop (material.blue.light).png
diff --git a/...atagrid/etalons/datagrid_filtering_by_range_3_desktop (material.blue.light).png b/...atagrid/etalons/datagrid_filtering_by_range_3_desktop (material.blue.light).png
diff --git a/...atagrid/etalons/datagrid_filtering_by_range_4_desktop (material.blue.light).png b/...atagrid/etalons/datagrid_filtering_by_range_4_desktop (material.blue.light).png
diff --git a/...s/datagrid/etalons/datagrid_focus_on_filter_1_desktop (material.blue.light).png b/...s/datagrid/etalons/datagrid_focus_on_filter_1_desktop (material.blue.light).png
diff --git a/...s/datagrid/etalons/datagrid_focus_on_filter_2_desktop (material.blue.light).png b/...s/datagrid/etalons/datagrid_focus_on_filter_2_desktop (material.blue.light).png
diff --git a/...s/datagrid/etalons/datagrid_focus_on_filter_3_desktop (material.blue.light).png b/...s/datagrid/etalons/datagrid_focus_on_filter_3_desktop (material.blue.light).png
diff --git a/apps/demos/utils/visual-tests/matrix-test-helper.js b/apps/demos/utils/visual-tests/matrix-test-helper.js
@@ -220,6 +220,7 @@ const SKIPPED_TESTS = {
       { demo: 'Overview', themes: [THEME.material] },
       { demo: 'ZoomingAndScrollingAPI', themes: [THEME.material] },
       { demo: 'ZoomingOnAreaSelection', themes: [THEME.material] },
+      { demo: 'CustomLegendMarkers', themes: [THEME.material] },
       { demo: 'DialogsAndNotificationsOverview', themes: [THEME.material] },
     ],
     VectorMap: [
@@ -245,6 +246,9 @@ const SKIPPED_TESTS = {
     Charts: [
       { demo: 'Overview', themes: [THEME.material] },
       { demo: 'ZoomingAndScrollingAPI', themes: [THEME.material] },
+      { demo: 'Crosshair', themes: [THEME.material] },
+      { demo: 'CustomAnnotations', themes: [THEME.material] },
+      { demo: 'CustomLegendMarkers', themes: [THEME.material] },
     ],
     DataGrid: [
       { demo: 'BatchEditing', themes: [THEME.fluent] },

diff --git a/e2e/testcafe-devextreme/helpers/themeUtils.ts b/e2e/testcafe-devextreme/helpers/themeUtils.ts
@@ -58,5 +58,7 @@ export async function testScreenshot(
       .ok();
   }
 
-  await changeTheme(process.env.theme ?? defaultThemeName);
+  if (isString(theme) || shouldTestInCompact) {
+    await changeTheme(process.env.theme ?? defaultThemeName);
+  }
 }
diff --git a/e2e/testcafe-devextreme/runner.js b/e2e/testcafe-devextreme/runner.js
@@ -59,7 +59,9 @@ createTestCafe({
             fs.rmSync('./screenshots', { recursive: true });
         }
 
-        const browsers = args.browsers.split(' ').map(expandBrowserAlias);
+        const browsers = args.browsers
+            .split(' ')
+            .map((browser) => expandBrowserAlias(browser, args.componentFolder.trim()));
         // eslint-disable-next-line no-console
         console.log('Browsers:', browsers);
 
@@ -113,24 +115,22 @@ createTestCafe({
             quarantineMode: { successThreshold: 1, attemptLimit: 5 },
         };
 
-        if(args.componentFolder.trim() !== 'renovation') {
-            runOptions.hooks = {
-                test: {
-                    before: async() => {
-                        if(args.shadowDom) {
-                            await addShadowRootTree();
-                        }
-
-                        if(args.theme) {
-                            await changeTheme(args.theme);
-                        }
-                    },
-                    after: async() => {
-                        await testPageUtils.clearTestPage();
+        runOptions.hooks = {
+            test: {
+                before: async() => {
+                    if(args.shadowDom) {
+                        await addShadowRootTree();
+                    }
+
+                    if(args.theme) {
+                        await changeTheme(args.theme);
                     }
                 },
-            };
-        }
+                after: async() => {
+                    await testPageUtils.clearTestPage();
+                }
+            },
+        };
 
         if(args.browsers === 'chrome:docker') {
             runOptions.disableScreenshots = true;
@@ -155,7 +155,7 @@ function setShadowDom(args) {
     process.env.shadowDom = args.shadowDom;
 }
 
-function expandBrowserAlias(browser) {
+function expandBrowserAlias(browser, componentFolder) {
     switch(browser) {
         case 'chrome:devextreme-shr2':
             return 'chrome:headless --disable-gpu --window-size=1200,800';

diff --git a/...vextreme/tests/dataGrid/etalons/disabled-toolbar-buttons-material.blue.dark.png b/...vextreme/tests/dataGrid/etalons/disabled-toolbar-buttons-material.blue.dark.png
diff --git a/...extreme/tests/dataGrid/etalons/disabled-toolbar-buttons-material.blue.light.png b/...extreme/tests/dataGrid/etalons/disabled-toolbar-buttons-material.blue.light.png
diff --git a/...id/focus/focusedRow/etalons/focused-row-invalid-cells (material.blue.light).png b/...id/focus/focusedRow/etalons/focused-row-invalid-cells (material.blue.light).png
diff --git a/.../dataGrid/focus/focusedRow/etalons/focused-row_markup (material.blue.light).png b/.../dataGrid/focus/focusedRow/etalons/focused-row_markup (material.blue.light).png
diff --git a/e2e/testcafe-devextreme/tests/editors/dateBox/dateBoxGeometry.ts b/e2e/testcafe-devextreme/tests/editors/dateBox/dateBoxGeometry.ts
@@ -15,9 +15,13 @@ safeSizeTest('Geometry is good', async (t) => {
   const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
   const dateBox = new DateBox('#container');
 
+  await dateBox.option('opened', true);
+
   await testScreenshot(t, takeScreenshot, 'Datebox with calendar.png');
 
+  await dateBox.option('opened', false);
   await dateBox.option('type', 'datetime');
+  await dateBox.option('opened', true);
 
   await testScreenshot(t, takeScreenshot, 'Datebox with datetime.png');
 
@@ -40,7 +44,6 @@ safeSizeTest('Geometry is good', async (t) => {
   await waitFont();
 
   return createWidget('dxDateBox', {
-    opened: true,
     pickerType: 'calendar',
     width: 200,
     value: new Date(1.5e12),

diff --git a/...s/DateBox with datetime and root element as container (material-blue-light).png b/...s/DateBox with datetime and root element as container (material-blue-light).png
diff --git a/...e is start in row, hover is end view & start row, rtl (material-blue-light).png b/...e is start in row, hover is end view & start row, rtl (material-blue-light).png
diff --git a/...B range, endDate is start in view & start in row, rtl (material-blue-light).png b/...B range, endDate is start in view & start in row, rtl (material-blue-light).png
diff --git a/...artDate inside row, hover is start in view & row, rtl (material-blue-light).png b/...artDate inside row, hover is start in view & row, rtl (material-blue-light).png
diff --git a/...sts/editors/dropDownBox/etalons/T1245111-dropDownBox-resize (generic-light).png b/...sts/editors/dropDownBox/etalons/T1245111-dropDownBox-resize (generic-light).png
diff --git a/...itors/dropDownBox/etalons/T1245111-dropDownBox-resize (material-blue-light).png b/...itors/dropDownBox/etalons/T1245111-dropDownBox-resize (material-blue-light).png
diff --git a/...em,item1_cSpan_2,item2_cSpan_1,location_left,cCount_1 (material-blue-light).png b/...em,item1_cSpan_2,item2_cSpan_1,location_left,cCount_1 (material-blue-light).png
diff --git a/...m,item1_cSpan_2,item2_cSpan_1,location_right,cCount_2 (material-blue-light).png b/...m,item1_cSpan_2,item2_cSpan_1,location_right,cCount_2 (material-blue-light).png
diff --git a/...tem,item1_cSpan_2,item2_cSpan_1,location_top,cCount_1 (material-blue-light).png b/...tem,item1_cSpan_2,item2_cSpan_1,location_top,cCount_1 (material-blue-light).png
diff --git a/...tem,item1_cSpan_2,item2_cSpan_1,location_top,cCount_2 (material-blue-light).png b/...tem,item1_cSpan_2,item2_cSpan_1,location_top,cCount_2 (material-blue-light).png
diff --git a/...em,item1_cSpan_2,item2_cSpan_2,location_left,cCount_2 (material-blue-light).png b/...em,item1_cSpan_2,item2_cSpan_2,location_left,cCount_2 (material-blue-light).png
diff --git a/...m,item1_cSpan_2,item2_cSpan_2,location_right,cCount_1 (material-blue-light).png b/...m,item1_cSpan_2,item2_cSpan_2,location_right,cCount_1 (material-blue-light).png
diff --git a/...tem,item1_cSpan_2,item2_cSpan_2,location_top,cCount_2 (material-blue-light).png b/...tem,item1_cSpan_2,item2_cSpan_2,location_top,cCount_2 (material-blue-light).png
diff --git a/...pleItem,rtl_false,location_left,cCount_3,itemsCount_1 (material-blue-light).png b/...pleItem,rtl_false,location_left,cCount_3,itemsCount_1 (material-blue-light).png
diff --git a/...leItem,rtl_false,location_right,cCount_2,itemsCount_1 (material-blue-light).png b/...leItem,rtl_false,location_right,cCount_2,itemsCount_1 (material-blue-light).png
diff --git a/...leItem,rtl_false,location_right,cCount_2,itemsCount_2 (material-blue-light).png b/...leItem,rtl_false,location_right,cCount_2,itemsCount_2 (material-blue-light).png
diff --git a/...mpleItem,rtl_false,location_top,cCount_1,itemsCount_1 (material-blue-light).png b/...mpleItem,rtl_false,location_top,cCount_1,itemsCount_1 (material-blue-light).png
diff --git a/...mpleItem,rtl_false,location_top,cCount_2,itemsCount_1 (material-blue-light).png b/...mpleItem,rtl_false,location_top,cCount_2,itemsCount_1 (material-blue-light).png
diff --git a/...mpleItem,rtl_true,location_left,cCount_1,itemsCount_2 (material-blue-light).png b/...mpleItem,rtl_true,location_left,cCount_1,itemsCount_2 (material-blue-light).png
diff --git a/...pleItem,rtl_true,location_right,cCount_1,itemsCount_1 (material-blue-light).png b/...pleItem,rtl_true,location_right,cCount_1,itemsCount_1 (material-blue-light).png
diff --git a/...impleItem,rtl_true,location_top,cCount_2,itemsCount_2 (material-blue-light).png b/...impleItem,rtl_true,location_top,cCount_2,itemsCount_2 (material-blue-light).png
diff --git a/...impleItem,rtl_true,location_top,cCount_3,itemsCount_1 (material-blue-light).png b/...impleItem,rtl_true,location_top,cCount_3,itemsCount_1 (material-blue-light).png