From 6067eb2153ad794c08d07fcc024dba73c3551db2 Mon Sep 17 00:00:00 2001 From: FRIN Yvonnick Date: Tue, 12 Nov 2019 16:03:05 +0100 Subject: [PATCH 1/5] :sparkles: Enable multiple organization for generate command --- src/core.js | 20 ++++++++++++++++---- src/index.js | 8 ++++---- src/utils.js | 5 +++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/core.js b/src/core.js index a5c82ce..0d9bc21 100644 --- a/src/core.js +++ b/src/core.js @@ -14,8 +14,15 @@ const START_ENHANCING_MEMBER = 'START_ENHANCING_MEMBER' const ENHANCING_MEMBER_DONE = 'ENHANCING_MEMBER_DONE' const ORGANIZATION_LOADED = 'ORGANIZATION_LOADED' -async function* generateOrganizationData(githubOrganization) { - const members = await getMembersByOrganization(githubOrganization) +async function* generateOrganizationData(githubOrganizations) { + let members = [] + + for (githubOrganization of githubOrganizations) { + const organizationMembers = await getMembersByOrganization(githubOrganization) + members = [...members, ...organizationMembers] + } + + members = filterNonUniqueBy(members, (a, b) => a.login === b.login) yield { type: MEMBERS_LOADED, value: members } for (member of members) { @@ -44,10 +51,15 @@ async function* generateOrganizationData(githubOrganization) { } } - const organization = await getRepositoriesByOrganization(githubOrganization) - yield { type: ORGANIZATION_LOADED, value: organization } + for (githubOrganization of githubOrganizations) { + const organization = await getRepositoriesByOrganization(githubOrganization) + yield { type: ORGANIZATION_LOADED, value: organization, name: githubOrganization } + } } +const filterNonUniqueBy = (arr, fn) => + arr.filter((v, i) => arr.every((x, j) => (i === j) === fn(v, x, i, j))); + module.exports = { generateOrganizationData, MEMBERS_LOADED, diff --git a/src/index.js b/src/index.js index 5182ba4..1b871da 100644 --- a/src/index.js +++ b/src/index.js @@ -28,14 +28,14 @@ require('yargs') }, }, async function(argv) { - const githubOrganization = argv.organization + const githubOrganizations = !Array.isArray(argv.organization) ? [argv.organization] : argv.organization const githubId = process.env.GITHUB_ID const githubToken = process.env.GITHUB_OAUTH await createDataFolder() - let bar - const generator = generateOrganizationData(githubOrganization) + let bar + const generator = generateOrganizationData(githubOrganizations) for await (const event of generator) { switch (event.type) { @@ -57,7 +57,7 @@ require('yargs') writeMember(event.value) break case ORGANIZATION_LOADED: - writeOrganization(event.value) + writeOrganization(event.name, event.value) break } } diff --git a/src/utils.js b/src/utils.js index 60a8319..b84405b 100644 --- a/src/utils.js +++ b/src/utils.js @@ -20,6 +20,7 @@ async function createDataFolder() { const folderPath = path.join(__dirname, rootDir) const alreadyExists = await existsAsync(folderPath) if (!alreadyExists) return mkdirAsync(folderPath) + .then(() => mkdirAsync(path.join(folderPath, 'organizations'))) } async function writeMember(member) { @@ -38,9 +39,9 @@ async function writeMembers(members) { return members } -async function writeOrganization(organization) { +async function writeOrganization(name, organization) { await writeFileAsync( - path.join(__dirname, rootDir, 'organization.json'), + path.join(__dirname, rootDir, 'organizations', `${name}.json`), prettyJson(organization), ) return organization From a23ef1261c2b4fab3c1a30e47a11ef4d96e7bcce Mon Sep 17 00:00:00 2001 From: FRIN Yvonnick Date: Tue, 12 Nov 2019 16:18:49 +0100 Subject: [PATCH 2/5] :sparkles: Enable multi-organisation in stats file --- src/stats.js | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/stats.js b/src/stats.js index 9450fd4..fed7a65 100644 --- a/src/stats.js +++ b/src/stats.js @@ -5,26 +5,23 @@ const chalk = require('chalk') const dataFolder = '../data' + const organizationsFolder = path.join(__dirname, dataFolder, 'organizations') const statsFile = 'stats.json' const generateFile = !!process.argv[2] const stats = {} - // Get the Gihub organization based on the .env values or the organization.json - stats.organization = - process.env.GITHUB_ORGA || - JSON.parse( - fs.readFileSync(path.join(__dirname, dataFolder, 'organization.json')), - )[0].owner.login + // Get the Gihub organization based on the .env values or the organization.json + if (process.env.GITHUB_ORGA) { + stats.organization = process.env.GITHUB_ORGA + } else { + const firstOrganizationFile = fs.readdirSync(organizationsFolder)[0] + console.log(path.join(organizationsFolder, firstOrganizationFile)) + stats.organization = JSON.parse(fs.readFileSync(path.join(organizationsFolder, firstOrganizationFile)))[0].owner.login + } - const members = fs - .readdirSync(path.join(__dirname, dataFolder)) - .filter( - file => - !['members.json', 'organization.json', 'stats.json'].includes(file), - ) - .map(file => - JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, file))), - ) + const members = fs.readdirSync(path.join(__dirname, dataFolder)) + .filter(file => !['members.json', 'organizations', 'stats.json'].includes(file)) + .map(file => JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, file)))) stats.totalMembers = members.length const membersWithRepositories = members.filter( @@ -74,9 +71,10 @@ .slice(0, 10) .map(([repo, count]) => ({ repo, count })) - const organizationRepositories = JSON.parse( - fs.readFileSync(path.join(__dirname, dataFolder, 'organization.json')), - ) + const organizationRepositories = fs.readdirSync(organizationsFolder) + .map(file => JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, 'organizations', file)))) + .flatMap(repositories => repositories) + stats.totalOrganizationRepositories = organizationRepositories.length stats.topOrganizationRepositories = organizationRepositories .reduce((acc, next) => { From 9f02410b8ce8854b2d64ee29f49199fa2c5badad Mon Sep 17 00:00:00 2001 From: FRIN Yvonnick Date: Tue, 26 Nov 2019 11:03:19 +0100 Subject: [PATCH 3/5] :construction: Improve organization file structure to have members --- src/core.js | 43 +++++++++++++++++++++++++++++++------------ src/stats.js | 31 +++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/core.js b/src/core.js index 0d9bc21..580e4f7 100644 --- a/src/core.js +++ b/src/core.js @@ -16,10 +16,14 @@ const ORGANIZATION_LOADED = 'ORGANIZATION_LOADED' async function* generateOrganizationData(githubOrganizations) { let members = [] + const membersByOrganisation = {} for (githubOrganization of githubOrganizations) { - const organizationMembers = await getMembersByOrganization(githubOrganization) - members = [...members, ...organizationMembers] + const organizationMembers = await getMembersByOrganization( + githubOrganization, + ) + membersByOrganisation[githubOrganization] = organizationMembers + members = [...members, ...organizationMembers] } members = filterNonUniqueBy(members, (a, b) => a.login === b.login) @@ -36,13 +40,17 @@ async function* generateOrganizationData(githubOrganizations) { for (repository of repositories) { await sleep(25) - const contributors = await getRepositoryContributors( - repository.owner.login, - repository.name, - ) - repository.contributors = contributors.map( - ({ weeks, ...contributor }) => contributor, - ) + try { + const contributors = await getRepositoryContributors( + repository.owner.login, + repository.name, + ) + repository.contributors = contributors.map( + ({ weeks, ...contributor }) => contributor, + ) + } catch (e) { + console.error('ERREUR', repository.owner.login, repository.name) + } } yield { @@ -52,13 +60,24 @@ async function* generateOrganizationData(githubOrganizations) { } for (githubOrganization of githubOrganizations) { - const organization = await getRepositoriesByOrganization(githubOrganization) - yield { type: ORGANIZATION_LOADED, value: organization, name: githubOrganization } + const organizationRepositories = await getRepositoriesByOrganization( + githubOrganization, + ) + const organization = { + name: githubOrganization, + members: membersByOrganisation[githubOrganization], + repositories: organizationRepositories, + } + yield { + type: ORGANIZATION_LOADED, + value: organization, + name: githubOrganization, + } } } const filterNonUniqueBy = (arr, fn) => - arr.filter((v, i) => arr.every((x, j) => (i === j) === fn(v, x, i, j))); + arr.filter((v, i) => arr.every((x, j) => (i === j) === fn(v, x, i, j))) module.exports = { generateOrganizationData, diff --git a/src/stats.js b/src/stats.js index fed7a65..9f3b821 100644 --- a/src/stats.js +++ b/src/stats.js @@ -10,18 +10,25 @@ const generateFile = !!process.argv[2] const stats = {} - // Get the Gihub organization based on the .env values or the organization.json + // Get the Gihub organization based on the .env values or the organization.json if (process.env.GITHUB_ORGA) { stats.organization = process.env.GITHUB_ORGA } else { const firstOrganizationFile = fs.readdirSync(organizationsFolder)[0] console.log(path.join(organizationsFolder, firstOrganizationFile)) - stats.organization = JSON.parse(fs.readFileSync(path.join(organizationsFolder, firstOrganizationFile)))[0].owner.login + stats.organization = JSON.parse( + fs.readFileSync(path.join(organizationsFolder, firstOrganizationFile)), + )[0].owner.login } - const members = fs.readdirSync(path.join(__dirname, dataFolder)) - .filter(file => !['members.json', 'organizations', 'stats.json'].includes(file)) - .map(file => JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, file)))) + const members = fs + .readdirSync(path.join(__dirname, dataFolder)) + .filter( + file => !['members.json', 'organizations', 'stats.json'].includes(file), + ) + .map(file => + JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, file))), + ) stats.totalMembers = members.length const membersWithRepositories = members.filter( @@ -71,10 +78,18 @@ .slice(0, 10) .map(([repo, count]) => ({ repo, count })) - const organizationRepositories = fs.readdirSync(organizationsFolder) - .map(file => JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, 'organizations', file)))) + const organizationRepositories = fs + .readdirSync(organizationsFolder) + .map(file => + JSON.parse( + fs.readFileSync( + path.join(__dirname, dataFolder, 'organizations', file), + ), + ), + ) + .map(organization => organization.repositories) .flatMap(repositories => repositories) - + stats.totalOrganizationRepositories = organizationRepositories.length stats.topOrganizationRepositories = organizationRepositories .reduce((acc, next) => { From 14fb530a359cc4a7eb23df9b2fa959db94e06943 Mon Sep 17 00:00:00 2001 From: FRIN Yvonnick Date: Tue, 26 Nov 2019 11:10:21 +0100 Subject: [PATCH 4/5] :recycle: Move organization logic --- src/core.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/core.js b/src/core.js index 580e4f7..002d6fa 100644 --- a/src/core.js +++ b/src/core.js @@ -29,6 +29,22 @@ async function* generateOrganizationData(githubOrganizations) { members = filterNonUniqueBy(members, (a, b) => a.login === b.login) yield { type: MEMBERS_LOADED, value: members } + for (githubOrganization of githubOrganizations) { + const organizationRepositories = await getRepositoriesByOrganization( + githubOrganization, + ) + const organization = { + name: githubOrganization, + members: membersByOrganisation[githubOrganization], + repositories: organizationRepositories, + } + yield { + type: ORGANIZATION_LOADED, + value: organization, + name: githubOrganization, + } + } + for (member of members) { yield { type: START_ENHANCING_MEMBER, value: member.login } @@ -58,22 +74,6 @@ async function* generateOrganizationData(githubOrganizations) { value: { ...member, repositories, contributionsCollection }, } } - - for (githubOrganization of githubOrganizations) { - const organizationRepositories = await getRepositoriesByOrganization( - githubOrganization, - ) - const organization = { - name: githubOrganization, - members: membersByOrganisation[githubOrganization], - repositories: organizationRepositories, - } - yield { - type: ORGANIZATION_LOADED, - value: organization, - name: githubOrganization, - } - } } const filterNonUniqueBy = (arr, fn) => From e047f74c3d2b3d9230ad46fc7efa4838b7c624f3 Mon Sep 17 00:00:00 2001 From: FRIN Yvonnick Date: Tue, 3 Dec 2019 09:52:29 +0100 Subject: [PATCH 5/5] :bug: Add a retry system for contributors --- src/core.js | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/core.js b/src/core.js index 002d6fa..2d663e5 100644 --- a/src/core.js +++ b/src/core.js @@ -56,17 +56,30 @@ async function* generateOrganizationData(githubOrganizations) { for (repository of repositories) { await sleep(25) - try { - const contributors = await getRepositoryContributors( + + let attempts = 0 + let contributors + + do { + if (attempts > 0) { + console.log('Retry fetch contributors for ', repository.name) + await sleep(1000) + } + contributors = await getRepositoryContributors( repository.owner.login, repository.name, ) - repository.contributors = contributors.map( - ({ weeks, ...contributor }) => contributor, - ) - } catch (e) { - console.error('ERREUR', repository.owner.login, repository.name) + attempts++ + } while (!Array.isArray(contributors) && attempts < 3) + + if (!Array.isArray(contributors)) { + console.log('Too much retry, set empty value: ', repository.name) + contributors = [] } + + repository.contributors = contributors.map( + ({ weeks, ...contributor }) => contributor, + ) } yield {