Fix cli broken parser (#81)

* Fix showInThread support for webhooks * Bump version to 1.1.58 in package.json * Fix really broken json results parsers * Add test to parse complicated json file * add suite name to the failure message
ryanrosello-og · Jan 6, 2024 · 3e836bf · 3e836bf
1 parent 080118f
commit 3e836bf
Show file tree

Hide file tree

Showing 14 changed files with 495 additions and 72 deletions.
diff --git a/package.json b/package.json
@@ -35,7 +35,7 @@
     "cli-debug": "yarn build && npx . -c ./cli_config.json -j ./tests/test_data/valid_test_results.json"
   },
   "name": "playwright-slack-report",
-  "version": "1.1.57",
+  "version": "1.1.58",
   "bin": {
     "playwright-slack-report": "dist/cli.js"
   },

diff --git a/src/LayoutGenerator.ts b/src/LayoutGenerator.ts
@@ -56,7 +56,7 @@ const generateFailures = async (
   );
 
   for (let i = 0; i < numberOfFailuresToShow; i += 1) {
-    const { failureReason, test } = summaryResults.failures[i];
+    const { failureReason, test, suite } = summaryResults.failures[i];
     const formattedFailure = failureReason
       .substring(0, maxNumberOfFailureLength)
       .split('\n')
@@ -66,7 +66,7 @@ const generateFailures = async (
       type: 'section',
       text: {
         type: 'mrkdwn',
-        text: `*${test}*
+        text: `*${suite} > ${test}*
         \n${formattedFailure}`,
       },
     });

diff --git a/src/ResultsParser.ts b/src/ResultsParser.ts
@@ -51,7 +51,10 @@ export default class ResultsParser {
     const parsedData: JSONResult = JSON.parse(data);
 
     const retries = parsedData.config.projects[0]?.retries || 0;
-    await this.parseTestSuite(parsedData.suites, retries);
+    for (const suite of parsedData.suites) {
+      // eslint-disable-next-line no-await-in-loop
+      await this.parseTestSuite(suite, retries);
+    }
 
     const failures = await this.getFailures();
     const summary: SummaryResults = {
@@ -66,44 +69,33 @@ export default class ResultsParser {
     for (const suite of this.result) {
       summary.tests = summary.tests.concat(suite.testSuite.tests);
     }
-    // console.log('🚀~ summary:', JSON.stringify(summary, null, 2));
 
     return summary;
   }
 
-  async parseTestSuite(suites: any, retries: number, suiteIndex = 0) {
+  async parseTestSuite(suites: any, retries: number) {
     let testResults = [];
-    if (suites[0].suites?.length > 0) {
-      testResults = await this.parseTests(
-        suites[0].title,
-        suites[0].specs,
-        retries,
-      );
-      this.updateResults({
-        testSuite: {
-          title: suites[0].title,
-          tests: testResults,
-        },
-      });
-      await this.parseTestSuite(
-        suites[suiteIndex].suites,
-        retries,
-        (suiteIndex += 1),
-      );
-    } else {
+
+    // if it has direct specs
+    if (suites.specs?.length > 0) {
       testResults = await this.parseTests(
-        suites[0].title,
-        suites[0].specs,
+        suites.title,
+        suites.specs,
         retries,
       );
       this.updateResults({
         testSuite: {
-          title: suites[0].title,
+          title: suites.title ?? suites.file,
           tests: testResults,
         },
       });
-      // eslint-disable-next-line no-useless-return
-      return;
+    }
+
+    if (suites.suites?.length > 0) {
+      for (const suite of suites.suites) {
+        // eslint-disable-next-line no-await-in-loop
+        await this.parseTestSuite(suite, retries);
+      }
     }
   }
 
@@ -175,6 +167,7 @@ export default class ResultsParser {
           // only flag as failed if the last attempt has failed
           if (test.retries === test.retry) {
             failures.push({
+              suite: test.suiteName,
               test: ResultsParser.getTestName(test),
               failureReason: test.reason,
             });

diff --git a/src/SlackReporter.ts b/src/SlackReporter.ts
@@ -189,6 +189,14 @@ class SlackReporter implements Reporter {
       };
     }
 
+    if (this.slackWebHookUrl && this.showInThread) {
+      return {
+        okToProceed: false,
+        message:
+          '❌ The showInThread feature is only supported when using slackOAuthToken or process.env.SLACK_BOT_USER_OAUTH_TOKEN',
+      };
+    }
+
     if (
       !this.sendResults
       || !['always', 'on-failure', 'off'].includes(this.sendResults)

diff --git a/src/cli/cli_pre_checks.ts b/src/cli/cli_pre_checks.ts
@@ -55,6 +55,14 @@ export const doPreChecks = async (
     };
   }
 
+  if (config.sendUsingWebhook && config.showInThread) {
+    return {
+      status: 'error',
+      message:
+        'The showInThread feature is only supported when using sendUsingBot is configured',
+    };
+  }
+
   if (!config.sendUsingWebhook && !config.sendUsingBot) {
     return {
       status: 'error',

diff --git a/src/index.ts b/src/index.ts
@@ -27,6 +27,7 @@ export type SummaryResults = {
 export type Meta = Array<{ key: string; value: string }>;
 
 export type failure = {
+  suite: string;
   test: string;
   failureReason: string;
 };

diff --git a/tests/ResultsParser.spec.ts b/tests/ResultsParser.spec.ts
@@ -508,7 +508,7 @@ test.describe('ResultsParser', () => {
     ).toEqual('Login [Project Name: nightly_regression] using chrome');
   });
 
-  test('parse test results from json file', async ({}) => {
+  test('parse test results from json file that has retries, flakies and skipped tests', async ({}) => {
     const resultsParser = new ResultsParser();
     const validTestResults = path.join(
       __dirname,
@@ -525,4 +525,22 @@ test.describe('ResultsParser', () => {
     expect(resultSummary.failures.length).toEqual(3);
     expect(resultSummary.tests.length).toEqual(10);
   });
+
+  test('parse test results from a complicated json file', async ({}) => {
+    const resultsParser = new ResultsParser();
+    const validTestResults = path.join(
+      __dirname,
+      'test_data',
+      'valid_test_results_complex.json',
+    );
+    const resultSummary = await resultsParser.parseFromJsonFile(
+      validTestResults,
+    );
+    expect(resultSummary.failed).toEqual(1);
+    expect(resultSummary.flaky).toEqual(0);
+    expect(resultSummary.skipped).toEqual(0);
+    expect(resultSummary.passed).toEqual(6);
+    expect(resultSummary.failures.length).toEqual(1);
+    expect(resultSummary.tests.length).toEqual(7);
+  });
 });
diff --git a/tests/SlackClient_generate_blocks.spec.ts b/tests/SlackClient_generate_blocks.spec.ts
@@ -10,9 +10,9 @@ test.describe('SlackClient.generateBlocks()', () => {
         flaky: undefined,
         skipped: 0,
         failures: [
-          { test: 'test', failureReason: 'message' },
-          { test: 'test2', failureReason: 'message' },
-          { test: 'test3', failureReason: 'message' },
+          { suite: 'smoke', test: 'test', failureReason: 'message' },
+          { suite: 'smoke', test: 'test2', failureReason: 'message' },
+          { suite: 'smoke', test: 'test3', failureReason: 'message' },
         ],
         tests: [],
       },
@@ -35,9 +35,9 @@ test.describe('SlackClient.generateBlocks()', () => {
         flaky: undefined,
         skipped: 0,
         failures: [
-          { test: 'test', failureReason: 'message' },
-          { test: 'test2', failureReason: 'message' },
-          { test: 'test3', failureReason: 'message' },
+          { suite: 'smoke', test: 'test', failureReason: 'message' },
+          { suite: 'smoke', test: 'test2', failureReason: 'message' },
+          { suite: 'smoke', test: 'test3', failureReason: 'message' },
         ],
         tests: [],
       },
@@ -59,7 +59,7 @@ test.describe('SlackClient.generateBlocks()', () => {
         passed: 1,
         flaky: undefined,
         skipped: 1,
-        failures: [{ test: 'test', failureReason: 'message' }],
+        failures: [{ suite: 'smoke', test: 'test', failureReason: 'message' }],
         tests: [],
       },
       2,
@@ -86,7 +86,7 @@ test.describe('SlackClient.generateBlocks()', () => {
         type: 'section',
         text: {
           type: 'mrkdwn',
-          text: '*test*\n        \n>message',
+          text: '*smoke > test*\n        \n>message',
         },
       },
     ]);
@@ -99,7 +99,7 @@ test.describe('SlackClient.generateBlocks()', () => {
         passed: 1,
         flaky: 1,
         skipped: 1,
-        failures: [{ test: 'test', failureReason: 'message' }],
+        failures: [{ suite: 'smoke', test: 'test', failureReason: 'message' }],
         tests: [],
       },
       2,
@@ -126,7 +126,7 @@ test.describe('SlackClient.generateBlocks()', () => {
         type: 'section',
         text: {
           type: 'mrkdwn',
-          text: '*test*\n        \n>message',
+          text: '*smoke > test*\n        \n>message',
         },
       },
     ]);

diff --git a/tests/SlackReporter.spec.ts b/tests/SlackReporter.spec.ts
@@ -261,6 +261,28 @@ test.describe('SlackReporter - preChecks()', () => {
     });
   });
 
+  test('showInThread only supported for bots not webhooks', async ({
+    fakeSlackReporter,
+    suite,
+    fullConfig,
+  }) => {
+    delete process.env.SLACK_BOT_USER_OAUTH_TOKEN;
+    const cloneFullConfig = JSON.parse(JSON.stringify(fullConfig));
+    cloneFullConfig.reporter = [
+      [
+        '/home/ry/_repo/playwright-slack-report/src/SlackReporter.ts',
+        { slackWebHookUrl: 'https://hooks.slack.com/services/1234', showInThread: true },
+      ],
+    ];
+    fakeSlackReporter.onBegin(cloneFullConfig, suite);
+
+    const result = fakeSlackReporter.preChecks();
+    expect(result).toEqual({
+      okToProceed: false,
+      message: '❌ The showInThread feature is only supported when using slackOAuthToken or process.env.SLACK_BOT_USER_OAUTH_TOKEN',
+    });
+  });
+
   test('okToProceed flag is set to false when the custom layout provided is not a Function', async ({
     fakeSlackReporter,
     suite,

diff --git a/tests/SlackWehookClient.spec.ts b/tests/SlackWehookClient.spec.ts
@@ -12,6 +12,7 @@ const test = base.extend<{ summaryResults: SummaryResults }>({
     skipped: 1,
     failures: [
       {
+        suite: 'smoke',
         test: 'test',
         failureReason: 'Unexpected error',
       },
@@ -86,7 +87,7 @@ test.describe('SlackWebhookClient.sendMessage()', () => {
           type: 'section',
           text: {
             type: 'mrkdwn',
-            text: '*test*\n        \n>Unexpected error',
+            text: '*smoke > test*\n        \n>Unexpected error',
           },
         },
       ],