diff --git a/README.md b/README.md index 6179f54..c489e99 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ A Github Action that can sync secrets from one repository to many others. This a **Required** New line deliminated regex expressions to select repositories. Repositires are limited to those in whcich the token user is an owner or collaborator. Set `repositories_list_regex` to `False` to use a hardcoded list of repositories. ### `repositories_list_regex` + If this value is `true` (default), the action will find all repositories available to the token user and filter based upon the regex provided. If it is false, it is expected that `repositories` will be an a @@ -32,6 +33,10 @@ new line deliminated list in the form of org/name. The number of retries to attempt when making Github calls when triggering rate limits or abuse limits. Defaults to 3. +### `concurrency` + +The number of allowed concurrent calls to the set secret endpoint. Lower this number to avoid abuse limits. Defaults to 10. + ### `dry_run` Run everything except for secret create and update functionality. @@ -48,6 +53,7 @@ uses: google/secrets-sync-action@v1.1.3 ${{github.repository}} DRY_RUN: true GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN_SECRETS }} + CONCURRENCY: 10 env: FOO: ${{github.run_id}} FOOBAR: BAZ diff --git a/__tests__/config.test.ts b/__tests__/config.test.ts index c178d03..f1485db 100644 --- a/__tests__/config.test.ts +++ b/__tests__/config.test.ts @@ -31,6 +31,7 @@ describe("getConfig", () => { const GITHUB_TOKEN = "token"; const DRY_RUN = false; const RETRIES = 3; + const CONCURRENCY = 50; const inputs = { INPUT_GITHUB_TOKEN: GITHUB_TOKEN, @@ -38,7 +39,8 @@ describe("getConfig", () => { INPUT_REPOSITORIES: REPOSITORIES.join("\n"), INPUT_REPOSITORIES_LIST_REGEX: String(REPOSITORIES_LIST_REGEX), INPUT_DRY_RUN: String(DRY_RUN), - INPUT_RETRIES: String(RETRIES) + INPUT_RETRIES: String(RETRIES), + INPUT_CONCURRENCY: String(CONCURRENCY) }; beforeEach(() => { @@ -62,7 +64,8 @@ describe("getConfig", () => { REPOSITORIES, REPOSITORIES_LIST_REGEX, DRY_RUN, - RETRIES + RETRIES, + CONCURRENCY }); }); diff --git a/__tests__/github.test.ts b/__tests__/github.test.ts index a1feeb7..d896e0b 100644 --- a/__tests__/github.test.ts +++ b/__tests__/github.test.ts @@ -21,7 +21,7 @@ import { filterReposByPatterns, listAllMatchingRepos, publicKeyCache, - setSecretsForRepo + setSecretForRepo } from "../src/github"; // @ts-ignore-next-line @@ -88,7 +88,7 @@ test("filterReposByPatterns matches patterns", async () => { expect(filterReposByPatterns([fixture[0].response], ["nope"]).length).toBe(0); }); -describe("setSecretsForRepo", () => { +describe("setSecretForRepo", () => { const repo = fixture[0].response; const publicKey = { key_id: "1234", @@ -117,19 +117,19 @@ describe("setSecretsForRepo", () => { .reply(200); }); - test("setSecretsForRepo should retrieve public key", async () => { - await setSecretsForRepo(octokit, secrets, repo, true); + test("setSecretForRepo should retrieve public key", async () => { + await setSecretForRepo(octokit, "FOO", secrets.FOO, repo, true); expect(publicKeyMock.isDone()).toBeTruthy(); }); - test("setSecretsForRepo should not set secret with dry run", async () => { - await setSecretsForRepo(octokit, secrets, repo, true); + test("setSecretForRepo should not set secret with dry run", async () => { + await setSecretForRepo(octokit, "FOO", secrets.FOO, repo, true); expect(publicKeyMock.isDone()).toBeTruthy(); expect(setSecretMock.isDone()).toBeFalsy(); }); - test("setSecretsForRepo should should call set secret endpoint", async () => { - await setSecretsForRepo(octokit, secrets, repo, false); + test("setSecretForRepo should should call set secret endpoint", async () => { + await setSecretForRepo(octokit, "FOO", secrets.FOO, repo, false); expect(nock.isDone()).toBeTruthy(); }); }); diff --git a/__tests__/main.test.ts b/__tests__/main.test.ts index 581c246..1e34ef6 100644 --- a/__tests__/main.test.ts +++ b/__tests__/main.test.ts @@ -32,7 +32,7 @@ test("run should succeed with a repo and secret", async () => { .fn() .mockImplementation(async () => [fixture[0].response]); - (github.setSecretsForRepo as jest.Mock) = jest + (github.setSecretForRepo as jest.Mock) = jest .fn() .mockImplementation(async () => null); @@ -46,12 +46,13 @@ test("run should succeed with a repo and secret", async () => { REPOSITORIES: [".*"], REPOSITORIES_LIST_REGEX: true, DRY_RUN: false, - RETRIES: 3 + RETRIES: 3, + CONCURRENCY: 1 }); await run(); expect(github.listAllMatchingRepos as jest.Mock).toBeCalledTimes(1); - expect((github.setSecretsForRepo as jest.Mock).mock.calls[0][2]).toEqual( + expect((github.setSecretForRepo as jest.Mock).mock.calls[0][3]).toEqual( fixture[0].response ); @@ -59,7 +60,7 @@ test("run should succeed with a repo and secret", async () => { }); test("run should succeed with a repo and secret with repository_list_regex as false", async () => { - (github.setSecretsForRepo as jest.Mock) = jest + (github.setSecretForRepo as jest.Mock) = jest .fn() .mockImplementation(async () => null); @@ -72,11 +73,12 @@ test("run should succeed with a repo and secret with repository_list_regex as fa SECRETS: ["BAZ"], REPOSITORIES: [fixture[0].response.full_name], REPOSITORIES_LIST_REGEX: false, - DRY_RUN: false + DRY_RUN: false, + CONCURRENCY: 1 }); await run(); - expect((github.setSecretsForRepo as jest.Mock).mock.calls[0][2]).toEqual({ + expect((github.setSecretForRepo as jest.Mock).mock.calls[0][3]).toEqual({ full_name: fixture[0].response.full_name }); diff --git a/action.yml b/action.yml index 47a8348..8403971 100644 --- a/action.yml +++ b/action.yml @@ -39,6 +39,12 @@ inputs: The number of retries to attempt when making Github calls. default: "3" required: false + concurrency: + description: | + The number of allowed concurrent calls to the set secret endpoint. Lower this + number to avoid abuse limits. + default: "10" + required: false runs: using: 'node12' main: 'dist/index.js' diff --git a/dist/index.js b/dist/index.js index bdab260..899a578 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1854,6 +1854,23 @@ const windowsRelease = release => { module.exports = windowsRelease; +/***/ }), + +/***/ 72: +/***/ (function(module) { + +"use strict"; + + +const pTry = (fn, ...arguments_) => new Promise(resolve => { + resolve(fn(...arguments_)); +}); + +module.exports = pTry; +// TODO: remove this in the next major version +module.exports.default = pTry; + + /***/ }), /***/ 87: @@ -1966,6 +1983,71 @@ module.exports.array = (stream, options) => getStream(stream, Object.assign({}, module.exports.MaxBufferError = MaxBufferError; +/***/ }), + +/***/ 158: +/***/ (function(module, __unusedexports, __webpack_require__) { + +"use strict"; + +const pTry = __webpack_require__(72); + +const pLimit = concurrency => { + if (!((Number.isInteger(concurrency) || concurrency === Infinity) && concurrency > 0)) { + return Promise.reject(new TypeError('Expected `concurrency` to be a number from 1 and up')); + } + + const queue = []; + let activeCount = 0; + + const next = () => { + activeCount--; + + if (queue.length > 0) { + queue.shift()(); + } + }; + + const run = (fn, resolve, ...args) => { + activeCount++; + + const result = pTry(fn, ...args); + + resolve(result); + + result.then(next, next); + }; + + const enqueue = (fn, resolve, ...args) => { + if (activeCount < concurrency) { + run(fn, resolve, ...args); + } else { + queue.push(run.bind(null, fn, resolve, ...args)); + } + }; + + const generator = (fn, ...args) => new Promise(resolve => enqueue(fn, resolve, ...args)); + Object.defineProperties(generator, { + activeCount: { + get: () => activeCount + }, + pendingCount: { + get: () => queue.length + }, + clearQueue: { + value: () => { + queue.length = 0; + } + } + }); + + return generator; +}; + +module.exports = pLimit; +module.exports.default = pLimit; + + /***/ }), /***/ 168: @@ -2101,11 +2183,15 @@ var __importStar = (this && this.__importStar) || function (mod) { result["default"] = mod; return result; }; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); const github_1 = __webpack_require__(824); const config_1 = __webpack_require__(478); const secrets_1 = __webpack_require__(545); +const p_limit_1 = __importDefault(__webpack_require__(158)); function run() { return __awaiter(this, void 0, void 0, function* () { try { @@ -2148,9 +2234,14 @@ function run() { FOUND_REPOS: repoNames, FOUND_SECRETS: Object.keys(secrets) }, null, 2)); + const limit = p_limit_1.default(config.CONCURRENCY); + const calls = []; for (const repo of repos) { - yield github_1.setSecretsForRepo(octokit, secrets, repo, config.DRY_RUN); + for (const k of Object.keys(secrets)) { + calls.push(limit(() => github_1.setSecretForRepo(octokit, k, secrets[k], repo, config.DRY_RUN))); + } } + yield Promise.all(calls); } catch (error) { /* istanbul ignore next */ @@ -5505,6 +5596,7 @@ const core = __importStar(__webpack_require__(470)); function getConfig() { const config = { GITHUB_TOKEN: core.getInput("GITHUB_TOKEN", { required: true }), + CONCURRENCY: Number(core.getInput("CONCURRENCY")), RETRIES: Number(core.getInput("RETRIES")), SECRETS: core.getInput("SECRETS", { required: true }).split("\n"), REPOSITORIES: core.getInput("REPOSITORIES", { required: true }).split("\n"), @@ -7509,14 +7601,6 @@ function getPublicKey(octokit, repo) { }); } exports.getPublicKey = getPublicKey; -function setSecretsForRepo(octokit, secrets, repo, dry_run) { - return __awaiter(this, void 0, void 0, function* () { - for (const k of Object.keys(secrets)) { - yield setSecretForRepo(octokit, k, secrets[k], repo, dry_run); - } - }); -} -exports.setSecretsForRepo = setSecretsForRepo; function setSecretForRepo(octokit, name, secret, repo, dry_run) { return __awaiter(this, void 0, void 0, function* () { const [repo_owner, repo_name] = repo.full_name.split("/"); diff --git a/package-lock.json b/package-lock.json index 3d35786..6ac1f8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5100,12 +5100,11 @@ "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" }, "p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", "requires": { - "p-try": "^1.0.0" + "p-try": "^2.0.0" } }, "p-locate": { @@ -5115,6 +5114,23 @@ "dev": true, "requires": { "p-limit": "^1.1.0" + }, + "dependencies": { + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true + } } }, "p-reduce": { @@ -5124,10 +5140,9 @@ "dev": true }, "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", - "dev": true + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, "parent-module": { "version": "1.0.1", diff --git a/package.json b/package.json index 8f3095e..d89b59f 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@octokit/plugin-retry": "^3.0.1", "@octokit/plugin-throttling": "^3.2.0", "@octokit/rest": "^17.1.0", + "p-limit": "^2.3.0", "tweetsodium": "0.0.4" }, "devDependencies": { diff --git a/src/config.ts b/src/config.ts index 6d54549..e7ea8c3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -23,11 +23,13 @@ export interface Config { REPOSITORIES_LIST_REGEX: boolean; DRY_RUN: boolean; RETRIES: number; + CONCURRENCY: number; } export function getConfig(): Config { const config = { GITHUB_TOKEN: core.getInput("GITHUB_TOKEN", { required: true }), + CONCURRENCY: Number(core.getInput("CONCURRENCY")), RETRIES: Number(core.getInput("RETRIES")), SECRETS: core.getInput("SECRETS", { required: true }).split("\n"), REPOSITORIES: core.getInput("REPOSITORIES", { required: true }).split("\n"), diff --git a/src/github.ts b/src/github.ts index 63b5601..bab949d 100644 --- a/src/github.ts +++ b/src/github.ts @@ -159,17 +159,6 @@ export async function getPublicKey( return publicKey; } -export async function setSecretsForRepo( - octokit: any, - secrets: { [key: string]: string }, - repo: Repository, - dry_run: boolean -): Promise { - for (const k of Object.keys(secrets)) { - await setSecretForRepo(octokit, k, secrets[k], repo, dry_run); - } -} - export async function setSecretForRepo( octokit: any, name: string, diff --git a/src/main.ts b/src/main.ts index a790ae6..5ca1571 100644 --- a/src/main.ts +++ b/src/main.ts @@ -20,11 +20,12 @@ import { DefaultOctokit, Repository, listAllMatchingRepos, - setSecretsForRepo + setSecretForRepo } from "./github"; import { getConfig } from "./config"; import { getSecrets } from "./secrets"; +import pLimit from "p-limit"; export async function run(): Promise { try { @@ -81,9 +82,19 @@ export async function run(): Promise { ) ); + const limit = pLimit(config.CONCURRENCY); + const calls: Promise[] = []; + for (const repo of repos) { - await setSecretsForRepo(octokit, secrets, repo, config.DRY_RUN); + for (const k of Object.keys(secrets)) { + calls.push( + limit(() => + setSecretForRepo(octokit, k, secrets[k], repo, config.DRY_RUN) + ) + ); + } } + await Promise.all(calls); } catch (error) { /* istanbul ignore next */ core.error(error);