✨ Enable passing multiple organizations to CLI

src/core.js

@@ -14,10 +14,37 @@ const START_ENHANCING_MEMBER = 'START_ENHANCING_MEMBER'
 const ENHANCING_MEMBER_DONE = 'ENHANCING_MEMBER_DONE'
 const ORGANIZATION_LOADED = 'ORGANIZATION_LOADED'
 
-async function* generateOrganizationData(githubOrganization) {
-  const members = await getMembersByOrganization(githubOrganization)
+async function* generateOrganizationData(githubOrganizations) {
+  let members = []
+  const membersByOrganisation = {}
+
+  for (githubOrganization of githubOrganizations) {
+    const organizationMembers = await getMembersByOrganization(
+      githubOrganization,
+    )
+    membersByOrganisation[githubOrganization] = organizationMembers
+    members = [...members, ...organizationMembers]
+  }
+
+  members = filterNonUniqueBy(members, (a, b) => a.login === b.login)
   yield { type: MEMBERS_LOADED, value: members }
 
+  for (githubOrganization of githubOrganizations) {
+    const organizationRepositories = await getRepositoriesByOrganization(
+      githubOrganization,
+    )
+    const organization = {
+      name: githubOrganization,
+      members: membersByOrganisation[githubOrganization],
+      repositories: organizationRepositories,
+    }
+    yield {
+      type: ORGANIZATION_LOADED,
+      value: organization,
+      name: githubOrganization,
+    }
+  }
+
   for (member of members) {
     yield { type: START_ENHANCING_MEMBER, value: member.login }
 
@@ -29,10 +56,27 @@ async function* generateOrganizationData(githubOrganization) {
 
     for (repository of repositories) {
       await sleep(25)
-      const contributors = await getRepositoryContributors(
-        repository.owner.login,
-        repository.name,
-      )
+
+      let attempts = 0
+      let contributors
+
+      do {
+        if (attempts > 0) {
+          console.log('Retry fetch contributors for ', repository.name)
+          await sleep(1000)
+        }
+        contributors = await getRepositoryContributors(
+          repository.owner.login,
+          repository.name,
+        )
+        attempts++
+      } while (!Array.isArray(contributors) && attempts < 3)
+
+      if (!Array.isArray(contributors)) {
+        console.log('Too much retry, set empty value: ', repository.name)
+        contributors = []
+      }
+
       repository.contributors = contributors.map(
         ({ weeks, ...contributor }) => contributor,
       )
@@ -43,11 +87,11 @@ async function* generateOrganizationData(githubOrganization) {
       value: { ...member, repositories, contributionsCollection },
     }
   }
-
-  const organization = await getRepositoriesByOrganization(githubOrganization)
-  yield { type: ORGANIZATION_LOADED, value: organization }
 }
 
+const filterNonUniqueBy = (arr, fn) =>
+  arr.filter((v, i) => arr.every((x, j) => (i === j) === fn(v, x, i, j)))
+
 module.exports = {
   generateOrganizationData,
   MEMBERS_LOADED,

src/index.js

@@ -28,14 +28,14 @@ require('yargs')
       },
     },
     async function(argv) {
-      const githubOrganization = argv.organization
+      const githubOrganizations = !Array.isArray(argv.organization) ? [argv.organization] : argv.organization
       const githubId = process.env.GITHUB_ID
       const githubToken = process.env.GITHUB_OAUTH
 
       await createDataFolder()
 
-      let bar
-      const generator = generateOrganizationData(githubOrganization)
+      let bar 
+      const generator = generateOrganizationData(githubOrganizations)
 
       for await (const event of generator) {
         switch (event.type) {
@@ -57,7 +57,7 @@ require('yargs')
             writeMember(event.value)
             break
           case ORGANIZATION_LOADED:
-            writeOrganization(event.value)
+            writeOrganization(event.name, event.value)
             break
         }
       }

src/stats.js

@@ -5,22 +5,26 @@
   const chalk = require('chalk')
 
   const dataFolder = '../data'
+  const organizationsFolder = path.join(__dirname, dataFolder, 'organizations')
   const statsFile = 'stats.json'
   const generateFile = !!process.argv[2]
   const stats = {}
 
   // Get the Gihub organization based on the .env values or the organization.json
-  stats.organization =
-    process.env.GITHUB_ORGA ||
-    JSON.parse(
-      fs.readFileSync(path.join(__dirname, dataFolder, 'organization.json')),
+  if (process.env.GITHUB_ORGA) {
+    stats.organization = process.env.GITHUB_ORGA
+  } else {
+    const firstOrganizationFile = fs.readdirSync(organizationsFolder)[0]
+    console.log(path.join(organizationsFolder, firstOrganizationFile))
+    stats.organization = JSON.parse(
+      fs.readFileSync(path.join(organizationsFolder, firstOrganizationFile)),
     )[0].owner.login
+  }
 
   const members = fs
     .readdirSync(path.join(__dirname, dataFolder))
     .filter(
-      file =>
-        !['members.json', 'organization.json', 'stats.json'].includes(file),
+      file => !['members.json', 'organizations', 'stats.json'].includes(file),
     )
     .map(file =>
       JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, file))),
@@ -74,9 +78,18 @@
     .slice(0, 10)
     .map(([repo, count]) => ({ repo, count }))
 
-  const organizationRepositories = JSON.parse(
-    fs.readFileSync(path.join(__dirname, dataFolder, 'organization.json')),
-  )
+  const organizationRepositories = fs
+    .readdirSync(organizationsFolder)
+    .map(file =>
+      JSON.parse(
+        fs.readFileSync(
+          path.join(__dirname, dataFolder, 'organizations', file),
+        ),
+      ),
+    )
+    .map(organization => organization.repositories)
+    .flatMap(repositories => repositories)
+
   stats.totalOrganizationRepositories = organizationRepositories.length
   stats.topOrganizationRepositories = organizationRepositories
     .reduce((acc, next) => {

src/utils.js

@@ -20,6 +20,7 @@ async function createDataFolder() {
   const folderPath = path.join(__dirname, rootDir)
   const alreadyExists = await existsAsync(folderPath)
   if (!alreadyExists) return mkdirAsync(folderPath)
+    .then(() => mkdirAsync(path.join(folderPath, 'organizations')))
 }
 
 async function writeMember(member) {
@@ -38,9 +39,9 @@ async function writeMembers(members) {
   return members
 }
 
-async function writeOrganization(organization) {
+async function writeOrganization(name, organization) {
   await writeFileAsync(
-    path.join(__dirname, rootDir, 'organization.json'),
+    path.join(__dirname, rootDir, 'organizations', `${name}.json`),
     prettyJson(organization),
   )
   return organization