test(e2e): restore report generation and fix location of received ima…

…ges (#3145) When an test failure occurred, the report was not generated and the development server did not stop at the end of the test execution. This was due to our custom function `toMatchImageSnapshot` becoming asynchronous in commit bffd25b. The original function is synchronous, so the custom function must be too. In addition to this fix, the logic of the code managing image attachments has been improved, mainly to better separate responsibilities. Storing the expected image copy and calculating the image location are specific to j-i-s, so there's no need to do this. to j-i-s, and are therefore performed first. The function for attaching images to the report only uses the previously calculated image location and is unaware that j-i-s is being used. For tests that redefined the location of output images (usually in a subfolder), the path to the received images was not updated accordingly. It was stored in the same location for each test, so only one file was kept and not stored with the diff and the expected image. Also regenerate the image for the “subprocess.03.collapsed.with.elements” test. - Previously, the test failed when run with firefox and this was for good reasons that were hidden with other browsers due to the threshold value used. - The sub-process on the top left is a "collapsed adhoc sub-process", so 2 markers must be displayed with a space between them. Currently, the adhoc marker is not displayed, but the expand marker is correctly positioned (not centered, but on the left) as if the adhoc marker were displayed. - Commit c0c50aa removed additional spacing between markers, so the expand marker has moved to the right. But the reference snapshot was not updated accordingly. - The test probably accepted a minor visual change like this, but at some point the new version of Firefox added other changes that are above the threshold, and the error now appears. - The problem didn't occur on macOS because the threshold was increased in commit bfa02fc, but this hid the problem.
process-analytics · Aug 21, 2024 · 7a9f2b4 · 7a9f2b4
1 parent be46a70
commit 7a9f2b4
Show file tree

Hide file tree

Showing 7 changed files with 159 additions and 95 deletions.
diff --git a/test/config/jest.image.ts b/test/config/jest.image.ts
@@ -25,7 +25,7 @@ import { toMatchImageSnapshot } from 'jest-image-snapshot';
 import MatcherContext = jest.MatcherContext;
 import CustomMatcherResult = jest.CustomMatcherResult;
 
-const jestLog = debugLogger('bv:test:jest:img');
+const log = debugLogger('bv:test:jest:img');
 const toMatchImageSnapshotWithRealSignature = toMatchImageSnapshot as (received: unknown, options?: MatchImageSnapshotOptions) => CustomMatcherResult;
 
 // The path is relative from the jest-html-reporters page to the folder storing the images
@@ -59,78 +59,99 @@ class RetriesCounter {
 
 const retriesCounter = new RetriesCounter();
 
-async function saveAndRegisterImages(matcherContext: MatcherContext, options: MatchImageSnapshotOptions): Promise<void> {
-  const snapshotIdentifier = options.customSnapshotIdentifier as string;
-  // Manage expected and received images
-  const baseImagePathWithName = `${options.customDiffDir}/${snapshotIdentifier}`;
-  const expectedImagePath = `${baseImagePathWithName}-expected.png`;
-  copyFileSync(`${options.customSnapshotsDir}/${snapshotIdentifier}.png`, expectedImagePath);
-  // this image is generated by jest-image-snapshot when activating `storeReceivedOnFailure`
-  const receivedImagePath = `${baseImagePathWithName}-received.png`;
-
+async function attachImagesForReport(images: LocationOfImagesForTestReport, matcherContext: MatcherContext): Promise<void> {
   // Attach the images to jest-html-reports
   // Chain the calls to preserve the attachment order
   // Create a custom context as the async call can be done whereas the global jest context has changed (another test is currently running).
   // So the test name and path changed, and the images would be attached to the wrong test.
   // For the context object structure, see https://github.com/Hazyzh/jest-html-reporters/blob/v3.0.5/helper.ts#L95
   const context: Record<symbol, unknown> = {};
+  const currentTestName = matcherContext.currentTestName;
+  log('Attaching images to report for test %s', currentTestName);
   context[Symbol('bpmn-visualization')] = {
     state: {
-      currentTestName: matcherContext.currentTestName,
+      currentTestName,
       testPath: matcherContext.testPath,
     },
     matchers: {}, // required by the jest-html-reporters getJestGlobalData function even if not used
   };
 
   try {
     await addAttach({
-      attach: computeRelativePathFromReportToSnapshots(`${baseImagePathWithName}-diff.png`),
+      attach: computeRelativePathFromReportToSnapshots(images.diff),
       description: 'diff',
       bufferFormat: 'png',
       context,
     });
 
     await addAttach({
-      attach: computeRelativePathFromReportToSnapshots(expectedImagePath),
+      attach: computeRelativePathFromReportToSnapshots(images.expected),
       description: 'expected',
       bufferFormat: 'png',
       context,
     });
 
     await addAttach({
-      attach: computeRelativePathFromReportToSnapshots(receivedImagePath),
+      attach: computeRelativePathFromReportToSnapshots(images.received),
       description: 'received',
       bufferFormat: 'png',
       context,
     });
+
+    log('Images attached to report for test %s', currentTestName);
   } catch (error) {
     console.error(
-      `Error while attaching images to test ${snapshotIdentifier}.` +
+      `Error while attaching images to test ${currentTestName}.` +
         `The 'jest-html-reporters' reporter is probably not in use. For instance, this occurs when running tests with the IntelliJ/Webstorm Jest runner.`,
       error,
     );
   }
 }
 
+type LocationOfImagesForTestReport = {
+  diff: string;
+  expected: string;
+  received: string;
+};
+
+function saveImages(options: MatchImageSnapshotOptions): LocationOfImagesForTestReport {
+  const snapshotIdentifier = options.customSnapshotIdentifier as string;
+  const baseImagePathWithName = `${options.customDiffDir}/${snapshotIdentifier}`;
+
+  const diffImagePath = `${baseImagePathWithName}-diff.png`;
+  const expectedImagePath = `${baseImagePathWithName}-expected.png`;
+  copyFileSync(`${options.customSnapshotsDir}/${snapshotIdentifier}.png`, expectedImagePath);
+  // this image is generated by jest-image-snapshot when activating `storeReceivedOnFailure`
+  const receivedImagePath = `${options.customReceivedDir}/${snapshotIdentifier}-received.png`;
+
+  return {
+    diff: diffImagePath,
+    expected: expectedImagePath,
+    received: receivedImagePath,
+  };
+}
+
 // Improve jest-image-snapshot outputs to facilitate debug
 // The 'options' parameter is mandatory for us, and some properties must be set as well
 // All options properties used here are always set in bpmn-visualization tests
 // If the following implementation would be done directly in jest-image-snapshot, this won't be required as it set default values we cannot access here
-async function toMatchImageSnapshotCustom(this: MatcherContext, received: Buffer, options: MatchImageSnapshotOptions): Promise<CustomMatcherResult> {
+function toMatchImageSnapshotCustom(this: MatcherContext, received: Buffer, options: MatchImageSnapshotOptions): CustomMatcherResult {
   const testId = this.currentTestName;
   retriesCounter.incrementExecutionCount(testId);
-  jestLog("Test: '%s' (test file path: '%s')", this.currentTestName, this.testPath);
+  log("Test: '%s' (test file path: '%s')", this.currentTestName, this.testPath);
   const executionCount = retriesCounter.getExecutionCount(testId);
-  jestLog('Ready to execute toMatchImageSnapshot, execution count: %s', executionCount);
+  log('Ready to execute toMatchImageSnapshot, execution count: %s', executionCount);
   const result = toMatchImageSnapshotWithRealSignature.call(this, received, options);
-  jestLog('toMatchImageSnapshot executed');
+  log('toMatchImageSnapshot executed');
 
   if (result.pass) {
-    jestLog('Result: success');
+    log('Result: success');
   } else {
-    jestLog('Result: failure');
+    log('Result: failure');
     if (retriesCounter.hasReachMaxRetries(testId)) {
-      await saveAndRegisterImages(this, options);
+      const imageLocationInformation = saveImages(options);
+      log('Images saved and ready to be registered to the report', imageLocationInformation);
+      attachImagesForReport(imageLocationInformation, this).catch(error => log('Fail to attach images to the report %s', error));
     }
 
     // Add configured failure threshold in the error message

diff --git a/...2e/__image_snapshots__/bpmn-rendering/subprocess.03.collapsed.with.elements.png b/...2e/__image_snapshots__/bpmn-rendering/subprocess.03.collapsed.with.elements.png
diff --git a/test/e2e/bpmn.rendering.test.ts b/test/e2e/bpmn.rendering.test.ts
@@ -165,12 +165,6 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds {
           windows: 0.66 / 100, // 0.6566433292574891%
         },
       ],
-      [
-        'subprocess.03.collapsed.with.elements',
-        {
-          macos: 0.11 / 100, // 0.10203186226079852%
-        },
-      ],
     ]);
   }
 

diff --git a/test/e2e/diagram.navigation.fit.test.ts b/test/e2e/diagram.navigation.fit.test.ts
@@ -19,9 +19,9 @@ import type { MatchImageSnapshotOptions } from 'jest-image-snapshot';
 
 import path from 'node:path';
 
-import 'jest-playwright-preset';
+import { ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds, withCustomOutputDirectory } from './helpers/visu/image-snapshot-config';
 
-import { ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds } from './helpers/visu/image-snapshot-config';
+import 'jest-playwright-preset';
 
 import { FitType } from '@lib/component/options';
 import { AvailableTestPages, PageTester } from '@test/shared/visu/bpmn-page-utils';
@@ -34,10 +34,13 @@ class FitImageSnapshotConfigurator extends ImageSnapshotConfigurator {
     fitType: FitType;
     margin?: number;
   }): MatchImageSnapshotOptions {
-    const config = super.getConfig(parameter);
-    config.customSnapshotsDir = FitImageSnapshotConfigurator.buildSnapshotFitDirectory(config.customSnapshotsDir, parameter.fitType, true, parameter.margin ?? 0);
-    config.customDiffDir = parameter.buildCustomDiffDir(config, parameter.fitType, parameter.margin);
-    return config;
+    return withCustomOutputDirectory(
+      {
+        ...super.getConfig(parameter),
+        customSnapshotsDir: FitImageSnapshotConfigurator.buildSnapshotFitDirectory(super.getConfig(parameter).customSnapshotsDir, parameter.fitType, true, parameter.margin ?? 0),
+      },
+      parameter.buildCustomDiffDir(super.getConfig(parameter), parameter.fitType, parameter.margin),
+    );
   }
 
   private static buildSnapshotFitDirectory(parentDirectory: string, fitType: FitType, withMargin = false, margin?: number): string {

diff --git a/test/e2e/diagram.navigation.zoom.pan.test.ts b/test/e2e/diagram.navigation.zoom.pan.test.ts
@@ -20,9 +20,9 @@ import type { Point } from '@test/shared/visu/bpmn-page-utils';
 import path from 'node:path';
 
 import debugLogger from 'debug';
-import 'jest-playwright-preset';
 
-import { ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds } from './helpers/visu/image-snapshot-config';
+import { withCustomOutputDirectory, ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds } from './helpers/visu/image-snapshot-config';
+import 'jest-playwright-preset';
 
 import { ZoomType } from '@lib/component/options';
 import { AvailableTestPages, PageTester } from '@test/shared/visu/bpmn-page-utils';
@@ -119,11 +119,15 @@ describe('diagram navigation - zoom and pan with mouse', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `mouse-zoom-in-out-${xTimes}-times`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `mouse-zoom-in-out-${xTimes}-times`),
+      ),
+    );
   });
 });
 
@@ -163,11 +167,15 @@ describe('diagram navigation - zoom with buttons', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `button-zoom-in-out-${xTimes}-times`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `button-zoom-in-out-${xTimes}-times`),
+      ),
+    );
   });
 });
 
@@ -205,11 +213,15 @@ describe('diagram navigation - zoom with buttons and mouse', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `zoom-button-then-mouse-${firstZoom}-then-${secondZoom}`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `zoom-button-then-mouse-${firstZoom}-then-${secondZoom}`),
+      ),
+    );
   });
 
   it.each`
@@ -222,10 +234,14 @@ describe('diagram navigation - zoom with buttons and mouse', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `zoom-mouse-then-button-${firstZoom}-then-${secondZoom}`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `zoom-mouse-then-button-${firstZoom}-then-${secondZoom}`),
+      ),
+    );
   });
 });
diff --git a/test/e2e/helpers/visu/image-snapshot-config.ts b/test/e2e/helpers/visu/image-snapshot-config.ts
@@ -55,14 +55,15 @@ export class ImageSnapshotConfigurator {
     const fileName = typeof parameter === 'string' ? parameter : parameter.fileName;
     const failureThreshold = this.getFailureThreshold(fileName);
 
-    return {
-      ...defaultImageSnapshotConfig,
-      failureThreshold: failureThreshold,
-      customSnapshotIdentifier: fileName,
-      customSnapshotsDir: this.defaultCustomSnapshotsDir,
-      customDiffDir: this.defaultCustomDiffDir,
-      customReceivedDir: this.defaultCustomDiffDir,
-    };
+    return withCustomOutputDirectory(
+      {
+        ...defaultImageSnapshotConfig,
+        failureThreshold: failureThreshold,
+        customSnapshotIdentifier: fileName,
+        customSnapshotsDir: this.defaultCustomSnapshotsDir,
+      },
+      this.defaultCustomDiffDir,
+    );
   }
 
   private getFailureThreshold(fileName: string): number {
@@ -166,3 +167,12 @@ export class MultiBrowserImageSnapshotThresholds {
     }
   }
 }
+
+/**
+ * Set both the customDiffDir and customReceivedDir to the same value.
+ */
+export const withCustomOutputDirectory = (options: MatchImageSnapshotOptions, customOutputDirectory: string): MatchImageSnapshotOptions => ({
+  ...options,
+  customDiffDir: customOutputDirectory,
+  customReceivedDir: customOutputDirectory,
+});