From fe7aab5e73b1d28f0045e6f4f2f24796706fd47d Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 30 Oct 2023 16:39:22 -0700 Subject: [PATCH] Make ./scripts a package, make scripts typecheck, remove old stuff, workflow updates (#67128) --- .github/workflows/CI.yml | 19 +++++ .github/workflows/UpdateCodeowners.yml | 11 ++- .github/workflows/format-and-commit.yml | 2 +- .github/workflows/ghostbuster.yml | 4 +- .github/workflows/lint-md.yml | 2 +- .github/workflows/support-window.yml | 2 +- package.json | 24 ++---- pnpm-workspace.yaml | 1 + scripts/clean-node-modules.js | 7 ++ scripts/close-old-issues.js | 65 --------------- scripts/fix-tslint.js | 1 + scripts/generate-tsconfigs.js | 41 ---------- scripts/ghostbuster.js | 25 +++++- scripts/jsconfig.json | 17 ---- scripts/package.json | 32 ++++++++ scripts/tsconfig.json | 1 + scripts/update-codeowners.js | 101 +++++++++++++----------- scripts/update-config/LintPackage.js | 4 +- scripts/update-config/index.js | 11 ++- scripts/update-config/updatePackage.js | 6 +- 20 files changed, 165 insertions(+), 211 deletions(-) delete mode 100644 scripts/close-old-issues.js delete mode 100644 scripts/generate-tsconfigs.js delete mode 100644 scripts/jsconfig.json create mode 100644 scripts/package.json diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0a12f54b1a203b..2fa45bc59a1bc0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -76,3 +76,22 @@ jobs: TOKEN='ghp_i5wtj1l2AbpFv3OU96w6R' TOKEN+='On3bHOkcV2AmVY6' DANGER_GITHUB_API_TOKEN=$TOKEN pnpm danger ci || $( exit 0 ) + + scripts: + runs-on: ubuntu-latest + if: github.repository == 'DefinitelyTyped/DefinitelyTyped' + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: '16' + + - uses: pnpm/action-setup@v2 + name: Install pnpm + with: + version: latest + run_install: | + - args: [--filter, ., --filter, '{./scripts}...'] + + - run: pnpm tsc -p ./scripts diff --git a/.github/workflows/UpdateCodeowners.yml b/.github/workflows/UpdateCodeowners.yml index 30865db35fd8d3..0718ddf7c2899e 100644 --- a/.github/workflows/UpdateCodeowners.yml +++ b/.github/workflows/UpdateCodeowners.yml @@ -31,11 +31,16 @@ jobs: with: version: latest run_install: | - - args: [--filter, .] + - args: [--filter, ., --filter, '{./scripts}...'] - run: git config --global user.email "typescriptbot@microsoft.com" - run: git config --global user.name "TypeScript Bot" - run: pnpm run update-codeowners - env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + + - uses: stefanzweifel/git-auto-commit-action@v5.0.0 + with: + commit_author: 'TypeScript Bot ' + commit_message: '🤖 Update CODEOWNERS' + commit_user_email: 'typescriptbot@microsoft.com' + commit_user_name: 'TypeScript Bot' diff --git a/.github/workflows/format-and-commit.yml b/.github/workflows/format-and-commit.yml index fdfbae5385460b..d38006b952c79c 100644 --- a/.github/workflows/format-and-commit.yml +++ b/.github/workflows/format-and-commit.yml @@ -25,7 +25,7 @@ jobs: with: version: latest run_install: | - - args: [--filter, .] + - args: [--filter, ., --filter, '{./scripts}...'] - name: Get date id: date diff --git a/.github/workflows/ghostbuster.yml b/.github/workflows/ghostbuster.yml index fd5efd31735421..40da258616f7fe 100644 --- a/.github/workflows/ghostbuster.yml +++ b/.github/workflows/ghostbuster.yml @@ -24,8 +24,6 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - uses: actions/setup-node@v4 @@ -34,7 +32,7 @@ jobs: with: version: latest run_install: | - - args: [--filter, .] + - args: [--filter, ., --filter, '{./scripts}...'] - run: node ./scripts/ghostbuster.js env: diff --git a/.github/workflows/lint-md.yml b/.github/workflows/lint-md.yml index 8017ab553ec8e7..3a6a4aec998296 100644 --- a/.github/workflows/lint-md.yml +++ b/.github/workflows/lint-md.yml @@ -17,5 +17,5 @@ jobs: with: version: latest run_install: | - - args: [--filter, .] + - args: [--filter, ., --filter, '{./scripts}...'] - run: pnpm remark --frail . .github diff --git a/.github/workflows/support-window.yml b/.github/workflows/support-window.yml index d1095c557a1a6f..91a39fb2336e16 100644 --- a/.github/workflows/support-window.yml +++ b/.github/workflows/support-window.yml @@ -31,7 +31,7 @@ jobs: with: version: latest run_install: | - - args: [--filter, .] + - args: [--filter, ., --filter, '{./scripts}...'] - name: Fetch TypeScript versions and release dates from npm run: | npm view --json typescript time | diff --git a/package.json b/package.json index aa36513d58581f..a39ce219a2c008 100644 --- a/package.json +++ b/package.json @@ -20,42 +20,30 @@ "preinstall": "npx only-allow pnpm", "not-needed": "node scripts/not-needed.js", "update-codeowners": "node scripts/update-codeowners.js", - "test-all": "node --require source-map-support/register node_modules/@definitelytyped/dtslint-runner/ --path .", + "test-all": "node --enable-source-maps node_modules/@definitelytyped/dtslint-runner/ --path .", "clean": "node scripts/remove-empty.js", "clean-node-modules": "node scripts/clean-node-modules.js", - "test": "node --require source-map-support/register node_modules/@definitelytyped/dtslint/ types", - "lint": "node --require source-map-support/register node_modules/@definitelytyped/dtslint/ types", + "test": "node --enable-source-maps node_modules/@definitelytyped/dtslint/ types", + "lint": "node --enable-source-maps node_modules/@definitelytyped/dtslint/ types", "prettier": "prettier" }, "devDependencies": { "@definitelytyped/definitions-parser": "latest", + "@definitelytyped/dts-critic": "latest", "@definitelytyped/dtslint": "latest", "@definitelytyped/dtslint-runner": "latest", "@definitelytyped/eslint-plugin": "latest", "@definitelytyped/header-parser": "latest", "@definitelytyped/typescript-versions": "latest", "@definitelytyped/utils": "latest", - "@octokit/core": "^3.5.1", - "@octokit/rest": "^16.0.0", - "comment-json": "^4.2.3", - "d3-array": "^3.0.2", - "d3-axis": "^3.0.0", - "d3-scale": "^4.0.0", - "d3-selection": "^3.0.0", - "d3-time": "^3.0.0", - "d3-time-format": "^4.0.0", "danger": "^11.2.3", "dprint": "^0.42.3", "eslint-plugin-jsdoc": "^44.2.7", - "jsdom": "^17.0.0", "remark-cli": "^11.0.0", "remark-gfm": "^3.0.0", "remark-validate-links": "^12.0.0", - "shelljs": "^0.8.5", - "source-map-support": "^0.5.21", - "typescript": "next", - "w3c-xmlserializer": "^2.0.0", - "yargs": "^17.1.1" + "tslint": "latest", + "typescript": "next" }, "type": "module", "husky": { diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 991b8ab2e5859d..fcc4e30544cf45 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,2 +1,3 @@ packages: + - 'scripts' - 'types/**' diff --git a/scripts/clean-node-modules.js b/scripts/clean-node-modules.js index d9f448631fa4bc..932f5d6d17bcf4 100644 --- a/scripts/clean-node-modules.js +++ b/scripts/clean-node-modules.js @@ -6,6 +6,10 @@ const __filename = url.fileURLToPath(new URL(import.meta.url)); const __dirname = path.dirname(__filename); const repoRoot = path.resolve(__dirname, ".."); +/** + * @param {string} p + * @returns {Iterable} + */ function* iterateNodeModules(p) { const dirents = fs.readdirSync(p, { withFileTypes: true }); for (const dirent of dirents) { @@ -20,6 +24,9 @@ function* iterateNodeModules(p) { } } +/** + * @param {string} p + */ function rimraf(p) { // The rimraf package uses maxRetries=10 on Windows, but Node's fs.rm does not have that special case. return fs.rmSync(p, { recursive: true, force: true, maxRetries: process.platform === "win32" ? 10 : 0 }); diff --git a/scripts/close-old-issues.js b/scripts/close-old-issues.js deleted file mode 100644 index ae6de806b5f5e4..00000000000000 --- a/scripts/close-old-issues.js +++ /dev/null @@ -1,65 +0,0 @@ -import { paginateRest } from "@octokit/plugin-paginate-rest"; -import { Octokit } from "@octokit/rest"; - -const CustomOctokit = Octokit.plugin(paginateRest); -const octokit = new CustomOctokit({ auth: process.env.GITHUB_API_TOKEN }); - -const message = ` -Hi thread, we're moving DefinitelyTyped to use [GitHub Discussions](https://github.com/DefinitelyTyped/DefinitelyTyped/issues/53377) for conversations the \`@types\` modules in DefinitelyTyped. - -To help with the transition, we're closing all issues which haven't had activity in the last 6 months, which includes this issue. If you think closing this issue is a mistake, please pop into the [TypeScript Community Discord](https://discord.gg/typescript) and mention the issue in the \`definitely-typed\` channel.`; - -const go = async () => { - const sixMonthsAgo = addMonths(new Date(), -6); - const dateQuery = sixMonthsAgo.toISOString().split("T")[0]; - - // For example: - // https://github.com/DefinitelyTyped/DefinitelyTyped/search?q=updated%3A%3C%3D2021-02-01&state=open&type=issues - - const parameters = { - per_page: 100, - q: `type:issue updated:<=${dateQuery} repo:DefinitelyTyped/DefinitelyTyped state:open`, - }; - - for await (const response of octokit.paginate.iterator("GET /search/issues", parameters)) { - console.log("\n-"); - /** @type {Array} */ - const items = response.data; - for (const issue of items) { - // Ignore issues with labels - if (issue.labels.length) continue; - - process.stdout.write("#" + issue.number + " "); - await octokit.issues.createComment({ - repo: "DefinitelyTyped", - owner: "DefinitelyTyped", - issue_number: issue.number, - body: message, - }); - await octokit.issues.update({ - repo: "DefinitelyTyped", - owner: "DefinitelyTyped", - issue_number: issue.number, - state: "closed", - }); - } - } -}; - -go(); - -// https://stackoverflow.com/questions/1648392/get-a-date-object-six-months-prior-from-another-date-object -function addMonths(date, months) { - var month = (date.getMonth() + months) % 12; - // create a new Date object that gets the last day of the desired month - var last = new Date(date.getFullYear(), month + 1, 0); - - // compare dates and set appropriately - if (date.getDate() <= last.getDate()) { - date.setMonth(month); - } else { - date.setMonth(month, last.getDate()); - } - - return date; -} diff --git a/scripts/fix-tslint.js b/scripts/fix-tslint.js index 6b5f47e30a9eb1..69b251e4e73802 100644 --- a/scripts/fix-tslint.js +++ b/scripts/fix-tslint.js @@ -29,6 +29,7 @@ for (const dirName of fs.readdirSync(home)) { function fixTslint(dir) { const target = new URL("tslint.json", dir); if (!fs.existsSync(target)) return; + /** @type {any} */ const json = JSON.parse(fs.readFileSync(target, "utf-8")); json.rules = fixRules(json.rules); const text = Object.keys(json).length === 1 diff --git a/scripts/generate-tsconfigs.js b/scripts/generate-tsconfigs.js deleted file mode 100644 index 1a1b61b47268fa..00000000000000 --- a/scripts/generate-tsconfigs.js +++ /dev/null @@ -1,41 +0,0 @@ -/// - -import * as fs from "node:fs"; - -const home = new URL("../types/", import.meta.url); - -for (const dirName of fs.readdirSync(home)) { - if (dirName.startsWith(".") || dirName === "node_modules" || dirName === "scripts") { - continue; - } - - const dir = new URL(`${dirName}/`, home); - const stats = fs.lstatSync(dir); - if (stats.isDirectory()) { - fixTsconfig(dir); - // Also do it for old versions - for (const subdir of fs.readdirSync(dir)) { - if (/^v\d+$/.test(subdir)) { - fixTsconfig(new URL(`${subdir}/`, dir)); - } - } - } -} - -/** - * @param {URL} dir - */ -function fixTsconfig(dir) { - const target = new URL("tsconfig.json", dir); - const json = JSON.parse(fs.readFileSync(target, "utf-8")); - json.compilerOptions = fixCompilerOptions(json.compilerOptions); - fs.writeFileSync(target, JSON.stringify(json, undefined, 4), "utf-8"); -} - -/** - * @param {{}} compilerOptions - */ -function fixCompilerOptions(compilerOptions) { - // Do something interesting here - return Object.fromEntries(Object.entries(compilerOptions).map(([key, value]) => [key, value])); -} diff --git a/scripts/ghostbuster.js b/scripts/ghostbuster.js index 18e669b04ed2ef..7894f0052efbde 100644 --- a/scripts/ghostbuster.js +++ b/scripts/ghostbuster.js @@ -1,8 +1,29 @@ -// @ts-check -import { flatMap, mapDefined } from "@definitelytyped/utils"; import { Octokit } from "@octokit/core"; import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +/** @type {(array: readonly T[] | undefined, mapfn: (x: T, i: number) => readonly U[]) => readonly U[]} */ +function flatMap(array, mapfn) { + const result = []; + if (array) { + for (let i = 0; i < array.length; i++) { + result.push(...mapfn(array[i], i)); + } + } + return result; +} + +/** @type {(arr: Iterable, mapper: (t: T) => U | undefined) => U[]} */ +function mapDefined(arr, mapper) { + const out = []; + for (const a of arr) { + const res = mapper(a); + if (res !== undefined) { + out.push(res); + } + } + return out; +} + /** * @typedef {{ githubUsername?: string }} Owner * @typedef {{ owners: Owner[]; raw: string; }} PackageInfo diff --git a/scripts/jsconfig.json b/scripts/jsconfig.json deleted file mode 100644 index 2a27ed812c535f..00000000000000 --- a/scripts/jsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "exclude": ["close-old-issues.js", "fix-tslint.js"], - "compilerOptions": { - "noUnusedLocals": true, - "target": "es6", - "module": "esnext", - "strict": true, - "baseUrl": "../types", - "moduleResolution": "node", - "resolveJsonModule": true, - "typeRoots": ["../types"], - "types": [], - "checkJs": true, - "forceConsistentCasingInFileNames": true, - "lib": ["es2019"] - } -} diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 00000000000000..6fa04df5156498 --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,32 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "@octokit/core": "^3.5.1", + "@types/d3-array": "^3.0.0", + "@types/d3-axis": "^3.0.0", + "@types/d3-scale": "^4.0.0", + "@types/d3-selection": "^3.0.0", + "@types/d3-time": "^3.0.0", + "@types/d3-time-format": "^4.0.0", + "@types/jsdom": "^21.1.0", + "@types/json-stable-stringify": "^1.0.0", + "@types/node": "*", + "@types/shelljs": "^0.8.0", + "@types/w3c-xmlserializer": "^2.0.0", + "@types/yargs": "^17.0.0", + "comment-json": "^4.2.3", + "d3-array": "^3.0.2", + "d3-axis": "^3.0.0", + "d3-scale": "^4.0.0", + "d3-selection": "^3.0.0", + "d3-time": "^3.0.0", + "d3-time-format": "^4.0.0", + "jsdom": "^17.0.0", + "json-stable-stringify": "^1.0.2", + "shelljs": "^0.8.5", + "typescript": "next", + "w3c-xmlserializer": "^2.0.0", + "yargs": "^17.1.1" + } +} diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index ce322ea0a75ce6..76de2451ad92f8 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -4,6 +4,7 @@ "module": "nodenext", "strict": true, "allowJs": true, + "checkJs": true, "noEmit": true }, "include": ["**/*.js"] diff --git a/scripts/update-codeowners.js b/scripts/update-codeowners.js index ad6732ff7fd2a4..94f4a7d952abdb 100644 --- a/scripts/update-codeowners.js +++ b/scripts/update-codeowners.js @@ -1,75 +1,80 @@ -// @ts-check -import { AllPackages, clean, getDefinitelyTyped, parseDefinitions } from "@definitelytyped/definitions-parser"; -import { loggerWithErrors } from "@definitelytyped/utils"; -import * as cp from "node:child_process"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import { writeFile } from "node:fs/promises"; -import * as os from "node:os"; -async function main() { - const options = { definitelyTypedPath: ".", progress: false, parseInParallel: true }; - const log = loggerWithErrors()[0]; +const header = `# This file is generated. +# Add yourself to the "owners" in package.json instead. +# See https://github.com/DefinitelyTyped/DefinitelyTyped#definition-owners`; - clean(); - const dt = await getDefinitelyTyped(options, log); - await parseDefinitions(dt, { nProcesses: os.cpus().length, definitelyTypedPath: "." }, log); - const allPackages = await AllPackages.read(dt); - const typings = allPackages.allTypings(); - const maxPathLen = Math.max(...typings.map(t => t.subDirectoryPath.length)); - const entries = mapDefined(typings, t => getEntry(t, maxPathLen)); +async function main() { + const { owners, maxPathLen } = getAllOwners(); + const codeOwnersPath = new URL("../.github/CODEOWNERS", import.meta.url); + const entries = mapDefined(owners, ([p, users]) => getEntry(p, users, maxPathLen)); await writeFile( - [options.definitelyTypedPath, ".github", "CODEOWNERS"].join("/"), + codeOwnersPath, `${header}\n\n${entries.join("\n")}\n`, { encoding: "utf-8" }, ); } -runSequence([ - ["git", ["checkout", "."]], // reset any changes -]); - main() - .then(() => { - runSequence([ - ["git", ["add", ".github/CODEOWNERS"]], // Add CODEOWNERS - ["git", ["pull"]], // Ensure we're up-to-date - ["git", ["commit", "-m", `"🤖 Update CODEOWNERS"`]], // Commit all changes - ["git", ["push"]], // push the branch - ]); - console.log(`Pushed new commit.`); - }) .catch(e => { console.error(e); process.exit(1); }); -/** @param {[string, string[]][]} tasks */ -function runSequence(tasks) { - for (const task of tasks) { - console.log(`${task[0]} ${task[1].join(" ")}`); - const result = cp.spawnSync(task[0], task[1], { timeout: 100000, shell: true, stdio: "inherit" }); - if (result.status !== 0) { - throw new Error(`${task[0]} ${task[1].join(" ")} failed: ${result.stderr && result.stderr.toString()}`); +/** + * @param {URL} dir + * @param {(subpath: URL) => void} fn + */ +function recurse(dir, fn) { + const entryPoints = readdirSync(dir, { withFileTypes: true }); + for (const subdir of entryPoints) { + if (subdir.isDirectory() && subdir.name !== "node_modules") { + const subpath = new URL(`${subdir.name}/`, dir); + fn(subpath); + recurse(subpath, fn); } } } -const header = `# This file is generated. -# Add yourself to the "Definitions by:" list instead. -# See https://github.com/DefinitelyTyped/DefinitelyTyped#definition-owners`; +function getAllOwners() { + /** @type {[string, string[]][]} */ + const owners = []; + console.log("Reading headers..."); + const rootPrefixLength = (new URL("../", import.meta.url)).pathname.length - 1; + let maxPathLen = 0; + + recurse(new URL("../types/", import.meta.url), subpath => { + const index = new URL("package.json", subpath); + if (existsSync(index)) { + const indexContent = readFileSync(index, "utf-8"); + let parsed; + try { + parsed = JSON.parse(indexContent); + } catch (e) {} + if (parsed && parsed.owners && Array.isArray(parsed.owners)) { + const usernames = mapDefined(parsed.owners, o => o.githubUsername); + if (usernames.length > 0) { + const p = subpath.pathname.slice(rootPrefixLength); + maxPathLen = Math.max(maxPathLen, p.length); + owners.push([p, usernames]); + } + } + } + }); + + return { maxPathLen, owners }; +} /** - * @param { import("@definitelytyped/definitions-parser").TypingsData } pkg + * @param {string} p + * @param {string[]} users * @param {number} maxPathLen * @return {string | undefined} */ -function getEntry(pkg, maxPathLen) { - const users = mapDefined(pkg.contributors, c => "githubUsername" in c ? c.githubUsername : undefined); - if (!users.length) { - return undefined; - } - - const path = `${pkg.subDirectoryPath}/`.padEnd(maxPathLen + 1); - return `/types/${path} ${users.map(u => `@${u}`).join(" ")}`; +function getEntry(p, users, maxPathLen) { + const path = p.padEnd(maxPathLen); + return `${path} ${users.map(u => `@${u}`).join(" ")}`; } /** diff --git a/scripts/update-config/LintPackage.js b/scripts/update-config/LintPackage.js index 697d3dd6f8ddb7..463c6794668395 100644 --- a/scripts/update-config/LintPackage.js +++ b/scripts/update-config/LintPackage.js @@ -1,8 +1,8 @@ -import * as stringify from "json-stable-stringify"; +import stringify from "json-stable-stringify"; import * as fs from "node:fs"; import * as path from "node:path"; import { Configuration as Config, Linter } from "tslint"; -import { isExternalDependency } from "./dependencies"; +import { isExternalDependency } from "./dependencies.js"; /** * Represents a package from the linter's perspective. diff --git a/scripts/update-config/index.js b/scripts/update-config/index.js index 4d83aec45ac033..ecb2501e82489a 100644 --- a/scripts/update-config/index.js +++ b/scripts/update-config/index.js @@ -10,13 +10,13 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { Configuration as Config } from "tslint"; -import * as yargs from "yargs"; -import { normalizePath } from "./dependencies"; -import { ignoredRules } from "./ignoredRules"; -import { updatePackage } from "./updatePackage"; +import yargs from "yargs"; +import { normalizePath } from "./dependencies.js"; +import { ignoredRules } from "./ignoredRules.js"; +import { updatePackage } from "./updatePackage.js"; async function main() { - const args = await yargs + const args = await yargs(process.argv.slice(2)) .usage(`\`$0 --dt=path-to-dt\` or \`$0 --package=path-to-dt-package\` 'dt.json' is used as the base tslint config for running the linter.`) .option("package", { @@ -39,7 +39,6 @@ async function main() { if (!arg.package && !arg.dt) { throw new Error("You must provide either argument 'package' or 'dt'."); } - // @ts-expect-error const unsupportedRules = arg.rules.filter(rule => ignoredRules.has(rule)); if (unsupportedRules.length > 0) { throw new Error(`Rules ${unsupportedRules.join(", ")} are not supported at the moment.`); diff --git a/scripts/update-config/updatePackage.js b/scripts/update-config/updatePackage.js index a5f7351fba2e76..4117795df148f7 100644 --- a/scripts/update-config/updatePackage.js +++ b/scripts/update-config/updatePackage.js @@ -2,9 +2,9 @@ import * as cp from "node:child_process"; import * as fs from "node:fs"; import * as path from "node:path"; -import { ignoredRules } from "./ignoredRules"; -import { LintPackage } from "./LintPackage"; -import { npmNamingDisabler } from "./npmNamingDisabler"; +import { ignoredRules } from "./ignoredRules.js"; +import { LintPackage } from "./LintPackage.js"; +import { npmNamingDisabler } from "./npmNamingDisabler.js"; /** * @param {string} pkgPath