feat: enable setting concurrency (#16)

README.md

@@ -19,6 +19,7 @@ A Github Action that can sync secrets from one repository to many others. This a
 **Required** New line deliminated regex expressions to select repositories. Repositires are limited to those in whcich the token user is an owner or collaborator. Set `repositories_list_regex` to `False` to use a hardcoded list of repositories.
 
 ### `repositories_list_regex`
+
 If this value is `true` (default), the action will find all
 repositories available to the token user and filter based upon the regex
 provided. If it is false, it is expected that `repositories` will be an a
@@ -32,6 +33,10 @@ new line deliminated list in the form of org/name.
 
 The number of retries to attempt when making Github calls when triggering rate limits or abuse limits. Defaults to 3.
 
+### `concurrency`
+
+The number of allowed concurrent calls to the set secret endpoint. Lower this number to avoid abuse limits. Defaults to 10.
+
 ### `dry_run`
 
 Run everything except for secret create and update functionality.
@@ -48,6 +53,7 @@ uses: google/[email protected]
       ${{github.repository}}
     DRY_RUN: true
     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN_SECRETS }}
+    CONCURRENCY: 10
   env:
     FOO: ${{github.run_id}}
     FOOBAR: BAZ

__tests__/config.test.ts

@@ -31,14 +31,16 @@ describe("getConfig", () => {
   const GITHUB_TOKEN = "token";
   const DRY_RUN = false;
   const RETRIES = 3;
+  const CONCURRENCY = 50;
 
   const inputs = {
     INPUT_GITHUB_TOKEN: GITHUB_TOKEN,
     INPUT_SECRETS: SECRETS.join("\n"),
     INPUT_REPOSITORIES: REPOSITORIES.join("\n"),
     INPUT_REPOSITORIES_LIST_REGEX: String(REPOSITORIES_LIST_REGEX),
     INPUT_DRY_RUN: String(DRY_RUN),
-    INPUT_RETRIES: String(RETRIES)
+    INPUT_RETRIES: String(RETRIES),
+    INPUT_CONCURRENCY: String(CONCURRENCY)
   };
 
   beforeEach(() => {
@@ -62,7 +64,8 @@ describe("getConfig", () => {
       REPOSITORIES,
       REPOSITORIES_LIST_REGEX,
       DRY_RUN,
-      RETRIES
+      RETRIES,
+      CONCURRENCY
     });
   });
 

__tests__/github.test.ts

@@ -21,7 +21,7 @@ import {
   filterReposByPatterns,
   listAllMatchingRepos,
   publicKeyCache,
-  setSecretsForRepo
+  setSecretForRepo
 } from "../src/github";
 
 // @ts-ignore-next-line
@@ -88,7 +88,7 @@ test("filterReposByPatterns matches patterns", async () => {
   expect(filterReposByPatterns([fixture[0].response], ["nope"]).length).toBe(0);
 });
 
-describe("setSecretsForRepo", () => {
+describe("setSecretForRepo", () => {
   const repo = fixture[0].response;
   const publicKey = {
     key_id: "1234",
@@ -117,19 +117,19 @@ describe("setSecretsForRepo", () => {
       .reply(200);
   });
 
-  test("setSecretsForRepo should retrieve public key", async () => {
-    await setSecretsForRepo(octokit, secrets, repo, true);
+  test("setSecretForRepo should retrieve public key", async () => {
+    await setSecretForRepo(octokit, "FOO", secrets.FOO, repo, true);
     expect(publicKeyMock.isDone()).toBeTruthy();
   });
 
-  test("setSecretsForRepo should not set secret with dry run", async () => {
-    await setSecretsForRepo(octokit, secrets, repo, true);
+  test("setSecretForRepo should not set secret with dry run", async () => {
+    await setSecretForRepo(octokit, "FOO", secrets.FOO, repo, true);
     expect(publicKeyMock.isDone()).toBeTruthy();
     expect(setSecretMock.isDone()).toBeFalsy();
   });
 
-  test("setSecretsForRepo should should call set secret endpoint", async () => {
-    await setSecretsForRepo(octokit, secrets, repo, false);
+  test("setSecretForRepo should should call set secret endpoint", async () => {
+    await setSecretForRepo(octokit, "FOO", secrets.FOO, repo, false);
     expect(nock.isDone()).toBeTruthy();
   });
 });

__tests__/main.test.ts

@@ -32,7 +32,7 @@ test("run should succeed with a repo and secret", async () => {
     .fn()
     .mockImplementation(async () => [fixture[0].response]);
 
-  (github.setSecretsForRepo as jest.Mock) = jest
+  (github.setSecretForRepo as jest.Mock) = jest
     .fn()
     .mockImplementation(async () => null);
 
@@ -46,20 +46,21 @@ test("run should succeed with a repo and secret", async () => {
     REPOSITORIES: [".*"],
     REPOSITORIES_LIST_REGEX: true,
     DRY_RUN: false,
-    RETRIES: 3
+    RETRIES: 3,
+    CONCURRENCY: 1
   });
   await run();
 
   expect(github.listAllMatchingRepos as jest.Mock).toBeCalledTimes(1);
-  expect((github.setSecretsForRepo as jest.Mock).mock.calls[0][2]).toEqual(
+  expect((github.setSecretForRepo as jest.Mock).mock.calls[0][3]).toEqual(
     fixture[0].response
   );
 
   expect(process.exitCode).toBe(undefined);
 });
 
 test("run should succeed with a repo and secret with repository_list_regex as false", async () => {
-  (github.setSecretsForRepo as jest.Mock) = jest
+  (github.setSecretForRepo as jest.Mock) = jest
     .fn()
     .mockImplementation(async () => null);
 
@@ -72,11 +73,12 @@ test("run should succeed with a repo and secret with repository_list_regex as fa
     SECRETS: ["BAZ"],
     REPOSITORIES: [fixture[0].response.full_name],
     REPOSITORIES_LIST_REGEX: false,
-    DRY_RUN: false
+    DRY_RUN: false,
+    CONCURRENCY: 1
   });
   await run();
 
-  expect((github.setSecretsForRepo as jest.Mock).mock.calls[0][2]).toEqual({
+  expect((github.setSecretForRepo as jest.Mock).mock.calls[0][3]).toEqual({
     full_name: fixture[0].response.full_name
   });
 

action.yml

@@ -39,6 +39,12 @@ inputs:
       The number of retries to attempt when making Github calls.
     default: "3"
     required: false
+  concurrency:
+    description: |
+      The number of allowed concurrent calls to the set secret endpoint. Lower this
+      number to avoid abuse limits.
+    default: "10"
+    required: false
 runs:
   using: 'node12'
   main: 'dist/index.js'

dist/index.js

@@ -1854,6 +1854,23 @@ const windowsRelease = release => {
 module.exports = windowsRelease;
 
 
+/***/ }),
+
+/***/ 72:
+/***/ (function(module) {
+
+"use strict";
+
+
+const pTry = (fn, ...arguments_) => new Promise(resolve => {
+	resolve(fn(...arguments_));
+});
+
+module.exports = pTry;
+// TODO: remove this in the next major version
+module.exports.default = pTry;
+
+
 /***/ }),
 
 /***/ 87:
@@ -1966,6 +1983,71 @@ module.exports.array = (stream, options) => getStream(stream, Object.assign({},
 module.exports.MaxBufferError = MaxBufferError;
 
 
+/***/ }),
+
+/***/ 158:
+/***/ (function(module, __unusedexports, __webpack_require__) {
+
+"use strict";
+
+const pTry = __webpack_require__(72);
+
+const pLimit = concurrency => {
+	if (!((Number.isInteger(concurrency) || concurrency === Infinity) && concurrency > 0)) {
+		return Promise.reject(new TypeError('Expected `concurrency` to be a number from 1 and up'));
+	}
+
+	const queue = [];
+	let activeCount = 0;
+
+	const next = () => {
+		activeCount--;
+
+		if (queue.length > 0) {
+			queue.shift()();
+		}
+	};
+
+	const run = (fn, resolve, ...args) => {
+		activeCount++;
+
+		const result = pTry(fn, ...args);
+
+		resolve(result);
+
+		result.then(next, next);
+	};
+
+	const enqueue = (fn, resolve, ...args) => {
+		if (activeCount < concurrency) {
+			run(fn, resolve, ...args);
+		} else {
+			queue.push(run.bind(null, fn, resolve, ...args));
+		}
+	};
+
+	const generator = (fn, ...args) => new Promise(resolve => enqueue(fn, resolve, ...args));
+	Object.defineProperties(generator, {
+		activeCount: {
+			get: () => activeCount
+		},
+		pendingCount: {
+			get: () => queue.length
+		},
+		clearQueue: {
+			value: () => {
+				queue.length = 0;
+			}
+		}
+	});
+
+	return generator;
+};
+
+module.exports = pLimit;
+module.exports.default = pLimit;
+
+
 /***/ }),
 
 /***/ 168:
@@ -2101,11 +2183,15 @@ var __importStar = (this && this.__importStar) || function (mod) {
     result["default"] = mod;
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const core = __importStar(__webpack_require__(470));
 const github_1 = __webpack_require__(824);
 const config_1 = __webpack_require__(478);
 const secrets_1 = __webpack_require__(545);
+const p_limit_1 = __importDefault(__webpack_require__(158));
 function run() {
     return __awaiter(this, void 0, void 0, function* () {
         try {
@@ -2148,9 +2234,14 @@ function run() {
                 FOUND_REPOS: repoNames,
                 FOUND_SECRETS: Object.keys(secrets)
             }, null, 2));
+            const limit = p_limit_1.default(config.CONCURRENCY);
+            const calls = [];
             for (const repo of repos) {
-                yield github_1.setSecretsForRepo(octokit, secrets, repo, config.DRY_RUN);
+                for (const k of Object.keys(secrets)) {
+                    calls.push(limit(() => github_1.setSecretForRepo(octokit, k, secrets[k], repo, config.DRY_RUN)));
+                }
             }
+            yield Promise.all(calls);
         }
         catch (error) {
             /* istanbul ignore next */
@@ -5505,6 +5596,7 @@ const core = __importStar(__webpack_require__(470));
 function getConfig() {
     const config = {
         GITHUB_TOKEN: core.getInput("GITHUB_TOKEN", { required: true }),
+        CONCURRENCY: Number(core.getInput("CONCURRENCY")),
         RETRIES: Number(core.getInput("RETRIES")),
         SECRETS: core.getInput("SECRETS", { required: true }).split("\n"),
         REPOSITORIES: core.getInput("REPOSITORIES", { required: true }).split("\n"),
@@ -7509,14 +7601,6 @@ function getPublicKey(octokit, repo) {
     });
 }
 exports.getPublicKey = getPublicKey;
-function setSecretsForRepo(octokit, secrets, repo, dry_run) {
-    return __awaiter(this, void 0, void 0, function* () {
-        for (const k of Object.keys(secrets)) {
-            yield setSecretForRepo(octokit, k, secrets[k], repo, dry_run);
-        }
-    });
-}
-exports.setSecretsForRepo = setSecretsForRepo;
 function setSecretForRepo(octokit, name, secret, repo, dry_run) {
     return __awaiter(this, void 0, void 0, function* () {
         const [repo_owner, repo_name] = repo.full_name.split("/");

package.json

@@ -29,6 +29,7 @@
     "@octokit/plugin-retry": "^3.0.1",
     "@octokit/plugin-throttling": "^3.2.0",
     "@octokit/rest": "^17.1.0",
+    "p-limit": "^2.3.0",
     "tweetsodium": "0.0.4"
   },
   "devDependencies": {

src/config.ts

@@ -23,11 +23,13 @@ export interface Config {
   REPOSITORIES_LIST_REGEX: boolean;
   DRY_RUN: boolean;
   RETRIES: number;
+  CONCURRENCY: number;
 }
 
 export function getConfig(): Config {
   const config = {
     GITHUB_TOKEN: core.getInput("GITHUB_TOKEN", { required: true }),
+    CONCURRENCY: Number(core.getInput("CONCURRENCY")),
     RETRIES: Number(core.getInput("RETRIES")),
     SECRETS: core.getInput("SECRETS", { required: true }).split("\n"),
     REPOSITORIES: core.getInput("REPOSITORIES", { required: true }).split("\n"),