From c8e5b2445a2188dfff5e087f77dda6367dd05e63 Mon Sep 17 00:00:00 2001 From: akshatnema Date: Fri, 11 Oct 2024 21:53:39 +0530 Subject: [PATCH 1/8] Removed promise from build-dashboard --- scripts/dashboard/build-dashboard.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index 49fbf8b5d4f..ee8cfc025b6 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -1,7 +1,6 @@ const { writeFileSync } = require('fs'); const { resolve } = require('path'); const { graphql } = require('@octokit/graphql'); -const { Promise } = require('node-fetch'); const { Queries } = require('./issue-queries'); async function getHotDiscussions(discussions) { @@ -163,4 +162,4 @@ async function start() { } start(); -module.exports = { getLabel, monthsSince, mapGoodFirstIssues, getHotDiscussions, getDiscussionByID } \ No newline at end of file +module.exports = { getLabel, monthsSince, mapGoodFirstIssues, getHotDiscussions, getDiscussionByID } From 2888676fd93de74c48d43b2261e29dd69fb64022 Mon Sep 17 00:00:00 2001 From: akshatnema Date: Sun, 13 Oct 2024 20:47:28 +0530 Subject: [PATCH 2/8] Updated script with pause in the scripts --- scripts/dashboard/build-dashboard.js | 219 +++++++++++++-------------- 1 file changed, 108 insertions(+), 111 deletions(-) diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index ee8cfc025b6..b48ac63536e 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -3,105 +3,20 @@ const { resolve } = require('path'); const { graphql } = require('@octokit/graphql'); const { Queries } = require('./issue-queries'); -async function getHotDiscussions(discussions) { - const result = await Promise.all( - discussions.map(async (discussion) => { - try { - const isPR = discussion.__typename === 'PullRequest'; - if (discussion.comments.pageInfo.hasNextPage) { - let fetchedDiscussion = await getDiscussionByID(isPR, discussion.id); - discussion = fetchedDiscussion.node; - } - - const interactionsCount = - discussion.reactions.totalCount + - discussion.comments.totalCount + - discussion.comments.nodes.reduce( - (acc, curr) => acc + curr.reactions.totalCount, - 0 - ); - - const finalInteractionsCount = isPR - ? interactionsCount + - discussion.reviews.totalCount + - discussion.reviews.nodes.reduce( - (acc, curr) => acc + curr.comments.totalCount, - 0 - ) - : interactionsCount; - return { - id: discussion.id, - isPR, - isAssigned: !!discussion.assignees.totalCount, - title: discussion.title, - author: discussion.author ? discussion.author.login : '', - resourcePath: discussion.resourcePath, - repo: 'asyncapi/' + discussion.repository.name, - labels: discussion.labels ? discussion.labels.nodes : [], - score: - finalInteractionsCount / - Math.pow(monthsSince(discussion.timelineItems.updatedAt) + 2, 1.8), - }; - } catch (e) { - console.error( - `there was some issues while parsing this item: ${JSON.stringify( - discussion - )}` - ); - throw e; - } - }) - ); - result.sort((ElemA, ElemB) => ElemB.score - ElemA.score); - const filteredResult = result.filter(issue => issue.author !== 'asyncapi-bot'); - return filteredResult.slice(0, 12); -} -async function writeToFile(content) { - writeFileSync( - resolve(__dirname, '..', '..', 'dashboard.json'), - JSON.stringify(content, null, ' ') - ); -} -async function mapGoodFirstIssues(issues) { - return issues.map((issue) => ({ - id: issue.id, - title: issue.title, - isAssigned: !!issue.assignees.totalCount, - resourcePath: issue.resourcePath, - repo: 'asyncapi/' + issue.repository.name, - author: issue.author.login, - area: getLabel(issue, 'area/') || 'Unknown', - labels: issue.labels.nodes.filter( - (label) => - !label.name.startsWith('area/') && - !label.name.startsWith('good first issue') - ), - })); -} - -function getLabel(issue, filter) { - const result = issue.labels.nodes.find((label) => - label.name.startsWith(filter) - ); - return result && result.name.split('/')[1]; -} - - -function monthsSince(date) { - const seconds = Math.floor((new Date() - new Date(date)) / 1000); - // 2592000 = number of seconds in a month = 30 * 24 * 60 * 60 - const months = seconds / 2592000; - return Math.floor(months); +async function pause(ms) { + return new Promise((res) => { + setTimeout(res, ms); + }); } async function getDiscussions(query, pageSize, endCursor = null) { try { - let result = await graphql(query, { + const result = await graphql(query, { first: pageSize, after: endCursor, headers: { - authorization: `token ${process.env.GITHUB_TOKEN}`, - }, + authorization: `token ${process.env.GITHUB_TOKEN}` + } }); if (result.rateLimit.remaining <= 100) { @@ -111,55 +26,137 @@ async function getDiscussions(query, pageSize, endCursor = null) { `limit = ${result.rateLimit.limit}`, `remaining = ${result.rateLimit.remaining}`, `resetAt = ${result.rateLimit.resetAt}` - ) + ); } - const hasNextPage = result.search.pageInfo.hasNextPage; + const { hasNextPage } = result.search.pageInfo; if (!hasNextPage) { return result.search.nodes; - } else { - return result.search.nodes.concat( - await getDiscussions(query, pageSize, result.search.pageInfo.endCursor) - ); } + return result.search.nodes.concat(await getDiscussions(query, pageSize, result.search.pageInfo.endCursor)); } catch (e) { console.error(e); + + return Promise.reject(e); } } async function getDiscussionByID(isPR, id) { try { - let result = await graphql(isPR ? Queries.pullRequestById : Queries.issueById, { + const result = await graphql(isPR ? Queries.pullRequestById : Queries.issueById, { id, headers: { - authorization: `token ${process.env.GITHUB_TOKEN}`, - }, + authorization: `token ${process.env.GITHUB_TOKEN}` + } + }); - } - ); return result; } catch (e) { console.error(e); + + return Promise.reject(e); + } +} + +async function getHotDiscussions(discussions) { + const result = []; + + for (let i = 0; i < discussions.length; i += 5) { + const batch = discussions.slice(i, i + 5); + // eslint-disable-next-line no-await-in-loop + const batchResults = await Promise.all( + batch.map(async (discussion) => { + try { + await pause(1000); + // eslint-disable-next-line no-underscore-dangle + const isPR = discussion.__typename === 'PullRequest'; + if (discussion.comments.pageInfo.hasNextPage) { + const fetchedDiscussion = await getDiscussionByID(isPR, discussion.id); + discussion = fetchedDiscussion.node; + } + + const interactionsCount = + discussion.reactions.totalCount + + discussion.comments.totalCount + + discussion.comments.nodes.reduce((acc, curr) => acc + curr.reactions.totalCount, 0); + + const finalInteractionsCount = isPR + ? interactionsCount + + discussion.reviews.totalCount + + discussion.reviews.nodes.reduce((acc, curr) => acc + curr.comments.totalCount, 0) + : interactionsCount; + return { + id: discussion.id, + isPR, + isAssigned: !!discussion.assignees.totalCount, + title: discussion.title, + author: discussion.author ? discussion.author.login : '', + resourcePath: discussion.resourcePath, + repo: `asyncapi/${discussion.repository.name}`, + labels: discussion.labels ? discussion.labels.nodes : [], + score: finalInteractionsCount / (monthsSince(discussion.timelineItems.updatedAt) + 2) ** 1.8 + }; + } catch (e) { + console.error(`there was some issues while parsing this item: ${JSON.stringify(discussion)}`); + throw e; + } + }) + ); + result.push(...batchResults); } + result.sort((ElemA, ElemB) => ElemB.score - ElemA.score); + const filteredResult = result.filter((issue) => issue.author !== 'asyncapi-bot'); + return filteredResult.slice(0, 12); +} +async function writeToFile(content) { + writeFileSync(resolve(__dirname, '..', '..', 'dashboard.json'), JSON.stringify(content, null, ' ')); +} +async function mapGoodFirstIssues(issues) { + return issues.map((issue) => ({ + id: issue.id, + title: issue.title, + isAssigned: !!issue.assignees.totalCount, + resourcePath: issue.resourcePath, + repo: `asyncapi/${issue.repository.name}`, + author: issue.author.login, + area: getLabel(issue, 'area/') || 'Unknown', + labels: issue.labels.nodes.filter( + (label) => !label.name.startsWith('area/') && !label.name.startsWith('good first issue') + ) + })); +} + +function getLabel(issue, filter) { + const result = issue.labels.nodes.find((label) => label.name.startsWith(filter)); + return result && result.name.split('/')[1]; } + +function monthsSince(date) { + const seconds = Math.floor((new Date() - new Date(date)) / 1000); + // 2592000 = number of seconds in a month = 30 * 24 * 60 * 60 + const months = seconds / 2592000; + return Math.floor(months); +} + async function start() { try { - const [issues, PRs, rawGoodFirstIssues] = await Promise.all([ - getDiscussions(Queries.hotDiscussionsIssues, 20), - getDiscussions(Queries.hotDiscussionsPullRequests, 20), - getDiscussions(Queries.goodFirstIssues, 20), - ]); + const issues = await getDiscussions(Queries.hotDiscussionsIssues, 20); + await pause(1000); + const PRs = await getDiscussions(Queries.hotDiscussionsPullRequests, 20); + await pause(1000); + const rawGoodFirstIssues = await getDiscussions(Queries.goodFirstIssues, 20); + await pause(1000); const discussions = issues.concat(PRs); const [hotDiscussions, goodFirstIssues] = await Promise.all([ getHotDiscussions(discussions), - mapGoodFirstIssues(rawGoodFirstIssues), + mapGoodFirstIssues(rawGoodFirstIssues) ]); writeToFile({ hotDiscussions, goodFirstIssues }); } catch (e) { - console.log('There were some issues parsing data from github.') + console.log('There were some issues parsing data from github.'); console.log(e); } } start(); -module.exports = { getLabel, monthsSince, mapGoodFirstIssues, getHotDiscussions, getDiscussionByID } +module.exports = { getLabel, monthsSince, mapGoodFirstIssues, getHotDiscussions, getDiscussionByID }; From 4058cedba11c7bd80731ed610afee5d406cd86c8 Mon Sep 17 00:00:00 2001 From: akshatnema Date: Sun, 20 Oct 2024 16:19:29 +0530 Subject: [PATCH 3/8] excluded some pause in the script --- scripts/dashboard/build-dashboard.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index b48ac63536e..210047fc593 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -67,7 +67,6 @@ async function getHotDiscussions(discussions) { const batchResults = await Promise.all( batch.map(async (discussion) => { try { - await pause(1000); // eslint-disable-next-line no-underscore-dangle const isPR = discussion.__typename === 'PullRequest'; if (discussion.comments.pageInfo.hasNextPage) { @@ -103,6 +102,9 @@ async function getHotDiscussions(discussions) { }) ); result.push(...batchResults); + + // eslint-disable-next-line no-await-in-loop + await pause(1000); } result.sort((ElemA, ElemB) => ElemB.score - ElemA.score); const filteredResult = result.filter((issue) => issue.author !== 'asyncapi-bot'); @@ -141,7 +143,6 @@ function monthsSince(date) { async function start() { try { const issues = await getDiscussions(Queries.hotDiscussionsIssues, 20); - await pause(1000); const PRs = await getDiscussions(Queries.hotDiscussionsPullRequests, 20); await pause(1000); const rawGoodFirstIssues = await getDiscussions(Queries.goodFirstIssues, 20); From d762073c6371ecc15588c043a496ef35b072750e Mon Sep 17 00:00:00 2001 From: akshatnema Date: Sun, 20 Oct 2024 18:15:39 +0530 Subject: [PATCH 4/8] updated dashboard to update the pause and add Promises --- .../workflows/regenerate-meetings-and-videos.yml | 8 ++++---- scripts/dashboard/build-dashboard.js | 15 +++++++-------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/regenerate-meetings-and-videos.yml b/.github/workflows/regenerate-meetings-and-videos.yml index 72aeda4e9e6..34509995d06 100644 --- a/.github/workflows/regenerate-meetings-and-videos.yml +++ b/.github/workflows/regenerate-meetings-and-videos.yml @@ -1,6 +1,6 @@ name: List everyday latest list of AsyncAPI Meetings, Newsroom Videos and Dashboard data. -on: +on: workflow_dispatch: schedule: #every day at midnight @@ -23,7 +23,7 @@ jobs: - name: Check package-lock version uses: asyncapi/.github/.github/actions/get-node-version-from-package-lock@master id: lockversion - + - name: Use Node.js uses: actions/setup-node@v3 with: @@ -45,7 +45,7 @@ jobs: committer: asyncapi-bot author: asyncapi-bot title: 'chore: update meetings.json and newsrooom_videos.json' - branch: update-meetings/${{ github.job }} + branch: update-meetings/${{ github.sha }} - if: failure() # Only, on failure, send a message on the 94_bot-failing-ci slack channel name: Report workflow run status to Slack uses: 8398a7/action-slack@fbd6aa58ba854a740e11a35d0df80cb5d12101d8 #using https://github.com/8398a7/action-slack/releases/tag/v3.15.1 @@ -54,4 +54,4 @@ jobs: fields: repo,action,workflow text: 'AsyncAPI Meetings and Videos workflow failed' env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_FAIL_NOTIFY }} \ No newline at end of file + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_FAIL_NOTIFY }} diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index 210047fc593..db18417e861 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -29,6 +29,8 @@ async function getDiscussions(query, pageSize, endCursor = null) { ); } + await pause(500); + const { hasNextPage } = result.search.pageInfo; if (!hasNextPage) { @@ -102,9 +104,6 @@ async function getHotDiscussions(discussions) { }) ); result.push(...batchResults); - - // eslint-disable-next-line no-await-in-loop - await pause(1000); } result.sort((ElemA, ElemB) => ElemB.score - ElemA.score); const filteredResult = result.filter((issue) => issue.author !== 'asyncapi-bot'); @@ -142,11 +141,11 @@ function monthsSince(date) { async function start() { try { - const issues = await getDiscussions(Queries.hotDiscussionsIssues, 20); - const PRs = await getDiscussions(Queries.hotDiscussionsPullRequests, 20); - await pause(1000); - const rawGoodFirstIssues = await getDiscussions(Queries.goodFirstIssues, 20); - await pause(1000); + const [issues, PRs, rawGoodFirstIssues] = await Promise.all([ + getDiscussions(Queries.hotDiscussionsIssues, 20), + getDiscussions(Queries.hotDiscussionsPullRequests, 20), + getDiscussions(Queries.goodFirstIssues, 20) + ]); const discussions = issues.concat(PRs); const [hotDiscussions, goodFirstIssues] = await Promise.all([ getHotDiscussions(discussions), From f2c1e1a849d9446668e20d2f94e48e4950ef5a69 Mon Sep 17 00:00:00 2001 From: akshatnema Date: Sun, 20 Oct 2024 18:34:12 +0530 Subject: [PATCH 5/8] Added extra pause in getHotDiscussions --- scripts/dashboard/build-dashboard.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index db18417e861..e3b349e68aa 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -103,6 +103,10 @@ async function getHotDiscussions(discussions) { } }) ); + + // eslint-disable-next-line no-await-in-loop + await pause(1000); + result.push(...batchResults); } result.sort((ElemA, ElemB) => ElemB.score - ElemA.score); From 0f14a3d405ea43563c889e12a92ba89a790096b5 Mon Sep 17 00:00:00 2001 From: akshatnema Date: Sun, 20 Oct 2024 18:58:10 +0530 Subject: [PATCH 6/8] added a optional chaining --- scripts/dashboard/build-dashboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index e3b349e68aa..c7041599127 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -133,7 +133,7 @@ async function mapGoodFirstIssues(issues) { function getLabel(issue, filter) { const result = issue.labels.nodes.find((label) => label.name.startsWith(filter)); - return result && result.name.split('/')[1]; + return result?.name.split('/')[1]; } function monthsSince(date) { From 339ac4f02c99cef40600843cd04fc63b307df750 Mon Sep 17 00:00:00 2001 From: akshatnema Date: Sun, 20 Oct 2024 22:56:20 +0530 Subject: [PATCH 7/8] removed Twitter embed from newsroom page --- components/newsroom/Newsroom.tsx | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/components/newsroom/Newsroom.tsx b/components/newsroom/Newsroom.tsx index 87b8bdff180..a88c4e2f14c 100644 --- a/components/newsroom/Newsroom.tsx +++ b/components/newsroom/Newsroom.tsx @@ -1,5 +1,4 @@ import React from 'react'; -import { TwitterTimelineEmbed } from 'react-twitter-embed'; import { HeadingLevel, HeadingTypeStyle } from '@/types/typography/Heading'; import { ParagraphTypeStyle } from '@/types/typography/Paragraph'; @@ -69,19 +68,14 @@ export default function Newsroom() { -
+
-
+
-
-
- -
-
From 433f38d690472e69fd61a9623e6690c07444b8a4 Mon Sep 17 00:00:00 2001 From: akshatnema Date: Sun, 20 Oct 2024 23:03:36 +0530 Subject: [PATCH 8/8] refactored build-dashboard script --- scripts/dashboard/build-dashboard.js | 88 ++++++++++++++++------------ 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index c7041599127..120109a05c2 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -3,6 +3,11 @@ const { resolve } = require('path'); const { graphql } = require('@octokit/graphql'); const { Queries } = require('./issue-queries'); +/** + * Introduces a delay in the execution flow. + * @param {number} ms - The number of milliseconds to pause. + * @returns {Promise} A promise that resolves after the specified delay. + */ async function pause(ms) { return new Promise((res) => { setTimeout(res, ms); @@ -60,49 +65,54 @@ async function getDiscussionByID(isPR, id) { } } +async function processHotDiscussions(batch) { + return Promise.all( + batch.map(async (discussion) => { + try { + // eslint-disable-next-line no-underscore-dangle + const isPR = discussion.__typename === 'PullRequest'; + if (discussion.comments.pageInfo.hasNextPage) { + const fetchedDiscussion = await getDiscussionByID(isPR, discussion.id); + discussion = fetchedDiscussion.node; + } + + const interactionsCount = + discussion.reactions.totalCount + + discussion.comments.totalCount + + discussion.comments.nodes.reduce((acc, curr) => acc + curr.reactions.totalCount, 0); + + const finalInteractionsCount = isPR + ? interactionsCount + + discussion.reviews.totalCount + + discussion.reviews.nodes.reduce((acc, curr) => acc + curr.comments.totalCount, 0) + : interactionsCount; + return { + id: discussion.id, + isPR, + isAssigned: !!discussion.assignees.totalCount, + title: discussion.title, + author: discussion.author ? discussion.author.login : '', + resourcePath: discussion.resourcePath, + repo: `asyncapi/${discussion.repository.name}`, + labels: discussion.labels ? discussion.labels.nodes : [], + score: finalInteractionsCount / (monthsSince(discussion.timelineItems.updatedAt) + 2) ** 1.8 + }; + } catch (e) { + console.error(`there was some issues while parsing this item: ${JSON.stringify(discussion)}`); + throw e; + } + }) + ); +} + async function getHotDiscussions(discussions) { const result = []; + const batchSize = 5; - for (let i = 0; i < discussions.length; i += 5) { - const batch = discussions.slice(i, i + 5); + for (let i = 0; i < discussions.length; i += batchSize) { + const batch = discussions.slice(i, i + batchSize); // eslint-disable-next-line no-await-in-loop - const batchResults = await Promise.all( - batch.map(async (discussion) => { - try { - // eslint-disable-next-line no-underscore-dangle - const isPR = discussion.__typename === 'PullRequest'; - if (discussion.comments.pageInfo.hasNextPage) { - const fetchedDiscussion = await getDiscussionByID(isPR, discussion.id); - discussion = fetchedDiscussion.node; - } - - const interactionsCount = - discussion.reactions.totalCount + - discussion.comments.totalCount + - discussion.comments.nodes.reduce((acc, curr) => acc + curr.reactions.totalCount, 0); - - const finalInteractionsCount = isPR - ? interactionsCount + - discussion.reviews.totalCount + - discussion.reviews.nodes.reduce((acc, curr) => acc + curr.comments.totalCount, 0) - : interactionsCount; - return { - id: discussion.id, - isPR, - isAssigned: !!discussion.assignees.totalCount, - title: discussion.title, - author: discussion.author ? discussion.author.login : '', - resourcePath: discussion.resourcePath, - repo: `asyncapi/${discussion.repository.name}`, - labels: discussion.labels ? discussion.labels.nodes : [], - score: finalInteractionsCount / (monthsSince(discussion.timelineItems.updatedAt) + 2) ** 1.8 - }; - } catch (e) { - console.error(`there was some issues while parsing this item: ${JSON.stringify(discussion)}`); - throw e; - } - }) - ); + const batchResults = await processHotDiscussions(batch); // eslint-disable-next-line no-await-in-loop await pause(1000);