Skip to content

Commit

Permalink
Publish older versions without touching latest tag
Browse files Browse the repository at this point in the history
  • Loading branch information
jablko committed Jun 1, 2022
1 parent cfaa4c0 commit d36ff5b
Show file tree
Hide file tree
Showing 19 changed files with 623 additions and 189 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,6 @@
"ts-jest": "^25.2.1",
"tslint": "^6.1.2",
"tslint-microsoft-contrib": "^6.2.0",
"typescript": "^4.5.5"
"typescript": "^4.6.4"
}
}
5 changes: 5 additions & 0 deletions packages/publisher/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,23 @@
"fs-extra": "^9.1.0",
"fstream": "^1.0.12",
"hh-mm-ss": "^1.2.0",
"libnpmpublish": "^6.0.4",
"longjohn": "^0.2.11",
"oboe": "^2.1.3",
"pacote": "^13.6.0",
"semver": "^7.3.7",
"source-map-support": "^0.4.0",
"typescript": "^4.1.0",
"yargs": "15.3.1"
},
"devDependencies": {
"@npm/types": "^1.0.2",
"@types/fs-extra": "^9.0.8",
"@types/hh-mm-ss": "^1.2.1",
"@types/libnpmpublish": "^4.0.3",
"@types/mz": "^0.0.31",
"@types/oboe": "^2.0.28",
"@types/pacote": "^11.1.4",
"@types/source-map-support": "^0.4.0",
"@types/yargs": "^15.0.4"
},
Expand Down
9 changes: 2 additions & 7 deletions packages/publisher/src/calculate-versions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,7 @@ async function computeAndSaveChangedPackages(
): Promise<ChangedPackages> {
const cp = await computeChangedPackages(allPackages, log, client);
const json: ChangedPackagesJson = {
changedTypings: cp.changedTypings.map(
({ pkg: { id }, version, latestVersion }): ChangedTypingJson => ({ id, version, latestVersion })
),
changedTypings: cp.changedTypings.map(({ pkg: { id }, version }): ChangedTypingJson => ({ id, version })),
changedNotNeededPackages: cp.changedNotNeededPackages.map((p) => p.name),
};
await writeDataFile(versionsFilename, json);
Expand All @@ -71,10 +69,7 @@ async function computeChangedPackages(
`'${pkg.name}' depends on '${name}' which does not exist on npm. All dependencies must exist.`
);
}
const latestVersion = pkg.isLatest
? undefined
: (await fetchTypesPackageVersionInfo(allPackages.getLatest(pkg), client, /*publish*/ true)).version;
return { pkg, version, latestVersion };
return { pkg, version };
}
return undefined;
});
Expand Down
4 changes: 2 additions & 2 deletions packages/publisher/src/generate-packages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import {
writeLog,
writeFile,
Logger,
writeTgz,
withNpmCache,
UncachedNpmInfoClient,
CachedNpmInfoClient,
Expand All @@ -34,6 +33,7 @@ import {
License,
formatTypingVersion,
} from "@definitelytyped/definitions-parser";
import * as pacote from "pacote";
import { readChangedPackages, ChangedPackages } from "./lib/versions";
import { outputDirectory } from "./util/util";
import { skipBadPublishes } from "./lib/npm";
Expand Down Expand Up @@ -65,7 +65,7 @@ export default async function generatePackages(
for (const { pkg, version } of changedPackages.changedTypings) {
await generateTypingPackage(pkg, allPackages, version, dt);
if (tgz) {
await writeTgz(outputDirectory(pkg), `${outputDirectory(pkg)}.tgz`);
await pacote.tarball.file(outputDirectory(pkg), `${outputDirectory(pkg)}.tgz`);
}
log(` * ${pkg.desc}`);
}
Expand Down
43 changes: 26 additions & 17 deletions packages/publisher/src/lib/package-publisher.ts
Original file line number Diff line number Diff line change
@@ -1,42 +1,51 @@
import assert = require("assert");
import { Logger, joinPaths, readFileAndWarn, NpmPublishClient } from "@definitelytyped/utils";
import { Logger, NpmPublishClient } from "@definitelytyped/utils";
import { NotNeededPackage, AnyPackage } from "@definitelytyped/definitions-parser";
import { updateTypeScriptVersionTags, updateLatestTag } from "@definitelytyped/retag";
import { updateTypeScriptVersionTags } from "@definitelytyped/retag";
import * as libpub from "libnpmpublish";
import * as pacote from "pacote";
import { ChangedTyping } from "./versions";
import { outputDirectory } from "../util/util";

// https://github.com/npm/types/pull/18
declare module "libnpmpublish" {
function publish(
manifest: Omit<import("@npm/types").PackageJson, "bundledDependencies">,
tarData: Buffer,
opts?: import("npm-registry-fetch").Options
): Promise<Response>;
}

export async function publishTypingsPackage(
client: NpmPublishClient,
changedTyping: ChangedTyping,
token: string,
dry: boolean,
log: Logger
): Promise<void> {
const { pkg, version, latestVersion } = changedTyping;
await common(client, pkg, log, dry);
const { pkg, version } = changedTyping;
await common(pkg, token, dry);
if (pkg.isLatest) {
await updateTypeScriptVersionTags(pkg, version, client, log, dry);
}
assert((latestVersion === undefined) === pkg.isLatest);
if (latestVersion !== undefined) {
// If this is an older version of the package, we still update tags for the *latest*.
// NPM will update "latest" even if we are publishing an older version of a package (https://github.com/npm/npm/issues/6778),
// so we must undo that by re-tagging latest.
await updateLatestTag(pkg.fullNpmName, latestVersion, client, log, dry);
}
}

export async function publishNotNeededPackage(
client: NpmPublishClient,
pkg: NotNeededPackage,
token: string,
dry: boolean,
log: Logger
): Promise<void> {
log(`Deprecating ${pkg.name}`);
await common(client, pkg, log, dry);
await common(pkg, token, dry);
}

async function common(client: NpmPublishClient, pkg: AnyPackage, log: Logger, dry: boolean): Promise<void> {
async function common(pkg: AnyPackage, token: string, dry: boolean): Promise<void> {
const packageDir = outputDirectory(pkg);
const packageJson = await readFileAndWarn("generate", joinPaths(packageDir, "package.json"));
await client.publish(packageDir, packageJson, dry, log);
const manifest = await pacote.manifest(packageDir).catch((cause) => {
throw cause.code === "ENOENT" ? new Error("Run generate first!", { cause }) : cause;
});
const tarData = await pacote.tarball(packageDir);
// Make sure we never assign the latest tag to an older version.
if (!dry)
await libpub.publish(manifest, tarData, { defaultTag: pkg.isLatest ? "latest" : "", access: "public", token });
}
6 changes: 1 addition & 5 deletions packages/publisher/src/lib/versions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ export interface ChangedTyping {
readonly pkg: TypingsData;
/** This is the version to be published, meaning it's the version that doesn't exist yet. */
readonly version: string;
/** For a non-latest version, this is the latest version; publishing an old version updates the 'latest' tag and we want to change it back. */
readonly latestVersion: string | undefined;
}

export interface ChangedPackagesJson {
Expand All @@ -25,7 +23,6 @@ export interface ChangedPackagesJson {
export interface ChangedTypingJson {
readonly id: PackageId;
readonly version: string;
readonly latestVersion?: string;
}

export interface ChangedPackages {
Expand All @@ -37,10 +34,9 @@ export async function readChangedPackages(allPackages: AllPackages): Promise<Cha
const json = (await readDataFile("calculate-versions", versionsFilename)) as ChangedPackagesJson;
return {
changedTypings: json.changedTypings.map(
({ id, version, latestVersion }): ChangedTyping => ({
({ id, version }): ChangedTyping => ({
pkg: allPackages.getTypingsData(id),
version,
latestVersion,
})
),
changedNotNeededPackages: json.changedNotNeededPackages.map((id) =>
Expand Down
7 changes: 4 additions & 3 deletions packages/publisher/src/publish-packages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,13 @@ export default async function publishPackages(
log("=== Publishing packages ===");
}

const client = await NpmPublishClient.create(await getSecret(Secret.NPM_TOKEN), undefined);
const token = await getSecret(Secret.NPM_TOKEN);
const client = await NpmPublishClient.create(token, undefined);

for (const cp of changedPackages.changedTypings) {
log(`Publishing ${cp.pkg.desc}...`);

await publishTypingsPackage(client, cp, dry, log);
await publishTypingsPackage(client, cp, token, dry, log);

const commits = (await queryGithub(
`repos/DefinitelyTyped/DefinitelyTyped/commits?path=types%2f${cp.pkg.subDirectoryPath}`,
Expand Down Expand Up @@ -134,7 +135,7 @@ export default async function publishPackages(
async (infoClient) => {
for (const n of changedPackages.changedNotNeededPackages) {
const target = skipBadPublishes(n, infoClient, log);
await publishNotNeededPackage(client, target, dry, log);
await publishNotNeededPackage(target, token, dry, log);
}
},
cacheDirPath
Expand Down
19 changes: 11 additions & 8 deletions packages/publisher/src/publish-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ import {
isObject,
max,
} from "@definitelytyped/utils";
import { PackageJson } from "@npm/types";
import * as libpub from "libnpmpublish";
import * as pacote from "pacote";
import * as semver from "semver";
import { getSecret, Secret } from "./lib/secrets";
// @ts-ignore
Expand Down Expand Up @@ -88,7 +91,7 @@ export default async function publishRegistry(

const token = await getSecret(Secret.NPM_TOKEN);

const publishClient = () => NpmPublishClient.create(token, { defaultTag: "next" });
const publishClient = () => NpmPublishClient.create(token);
if (!semver.eq(highestSemverVersion, npmVersion)) {
// There was an error in the last publish and types-registry wasn't validated.
// This may have just been due to a timeout, so test if types-registry@next is a subset of the one we're about to publish.
Expand All @@ -98,7 +101,7 @@ export default async function publishRegistry(
await (await publishClient()).tag(typesRegistry, String(highestSemverVersion), "latest", dry, log);
} else if (npmContentHash !== newContentHash && isTimeForNewVersion) {
log("New packages have been added, so publishing a new registry.");
await publish(await publishClient(), typesRegistry, packageJson, newVersion, dry, log);
await publish(await publishClient(), packageJson, token, dry, log);
} else {
const reason =
npmContentHash === newContentHash ? "No new packages published" : "Was modified less than a week ago";
Expand Down Expand Up @@ -137,13 +140,13 @@ async function generate(registry: string, packageJson: {}): Promise<void> {

async function publish(
client: NpmPublishClient,
packageName: string,
packageJson: {},
version: string,
packageJson: PackageJson,
token: string,
dry: boolean,
log: Logger
): Promise<void> {
await client.publish(registryOutputPath, packageJson, dry, log);
const tarData = await pacote.tarball(registryOutputPath);
if (!dry) await libpub.publish(packageJson, tarData, { defaultTag: "next", access: "public", token });
// Sleep for 60 seconds to let NPM update.
if (dry) {
log("(dry) Skipping 60 second sleep...");
Expand All @@ -153,7 +156,7 @@ async function publish(
}
// Don't set it as "latest" until *after* it's been validated.
await validate(log);
await client.tag(packageName, version, "latest", dry, log);
await client.tag(packageJson.name, packageJson.version, "latest", dry, log);
}

async function installForValidate(log: Logger): Promise<void> {
Expand Down Expand Up @@ -228,7 +231,7 @@ function assertJsonNewer(newer: { [s: string]: any }, older: { [s: string]: any
}
}

function generatePackageJson(name: string, version: string, typesPublisherContentHash: string): object {
function generatePackageJson(name: string, version: string, typesPublisherContentHash: string): PackageJson {
const json = {
name,
version,
Expand Down
47 changes: 47 additions & 0 deletions packages/publisher/test/package-publisher.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { NotNeededPackage, TypingsData } from "@definitelytyped/definitions-parser";
import * as libpub from "libnpmpublish";
import { publishNotNeededPackage, publishTypingsPackage } from "../src/lib/package-publisher";

jest.mock("libnpmpublish");
jest.mock("pacote", () => ({ async manifest() {}, async tarball() {} }));

const client = { async tag() {} };

const latestVersion = {
pkg: new TypingsData({ typingsPackageName: "some-package" } as never, /*isLatest*/ true),
version: "1.2.3",
};
const olderVersion = {
pkg: new TypingsData({ typingsPackageName: "some-package" } as never, /*isLatest*/ false),
version: "1.2.3",
};
const notNeeded = new NotNeededPackage("not-needed", "not-needed", "1.2.3");

function log() {}

test("Latest version gets latest tag", async () => {
await publishTypingsPackage(client as never, latestVersion, /*token*/ "", /*dry*/ false, log);
expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
defaultTag: "latest",
access: "public",
token: "",
});
});

test("Publishing older versions doesn't change the latest tag", async () => {
await publishTypingsPackage(client as never, olderVersion, /*token*/ "", /*dry*/ false, log);
expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
defaultTag: "",
access: "public",
token: "",
});
});

test("Not-needed types package gets latest tag", async () => {
await publishNotNeededPackage(notNeeded, /*token*/ "", /*dry*/ false, log);
expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
defaultTag: "latest",
access: "public",
token: "",
});
});
1 change: 0 additions & 1 deletion packages/utils/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
logs
test/data/pack.tgz
1 change: 0 additions & 1 deletion packages/utils/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
"charm": "^1.0.2",
"fs-extra": "^8.1.0",
"fstream": "^1.0.12",
"tar": "^6.1.11",
"tar-stream": "^2.1.4"
},
"devDependencies": {
Expand Down
Loading

0 comments on commit d36ff5b

Please sign in to comment.