Skip to content

Commit

Permalink
Publish old versions without touching latest tag
Browse files Browse the repository at this point in the history
  • Loading branch information
jablko committed Jun 29, 2022
1 parent 4011b70 commit 383193f
Show file tree
Hide file tree
Showing 18 changed files with 120 additions and 199 deletions.
3 changes: 3 additions & 0 deletions packages/publisher/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"fs-extra": "^9.1.0",
"fstream": "^1.0.12",
"hh-mm-ss": "^1.2.0",
"libnpmpublish": "^6.0.4",
"longjohn": "^0.2.11",
"oboe": "^2.1.3",
"pacote": "^13.6.1",
Expand All @@ -28,8 +29,10 @@
"yargs": "15.3.1"
},
"devDependencies": {
"@npm/types": "^1.0.2",
"@types/fs-extra": "^9.0.8",
"@types/hh-mm-ss": "^1.2.1",
"@types/libnpmpublish": "^4.0.3",
"@types/mz": "^0.0.31",
"@types/oboe": "^2.0.28",
"@types/pacote": "^11.1.5",
Expand Down
9 changes: 2 additions & 7 deletions packages/publisher/src/calculate-versions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,7 @@ async function computeAndSaveChangedPackages(
): Promise<ChangedPackages> {
const cp = await computeChangedPackages(allPackages, log);
const json: ChangedPackagesJson = {
changedTypings: cp.changedTypings.map(
({ pkg: { id }, version, latestVersion }): ChangedTypingJson => ({ id, version, latestVersion })
),
changedTypings: cp.changedTypings.map(({ pkg: { id }, version }): ChangedTypingJson => ({ id, version })),
changedNotNeededPackages: cp.changedNotNeededPackages.map((p) => p.name),
};
await writeDataFile(versionsFilename, json);
Expand All @@ -63,10 +61,7 @@ async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithE
: reason;
});
}
const latestVersion = pkg.isLatest
? undefined
: (await fetchTypesPackageVersionInfo(allPackages.getLatest(pkg), /*publish*/ true)).version;
return { pkg, version, latestVersion };
return { pkg, version };
}
return undefined;
});
Expand Down
3 changes: 1 addition & 2 deletions packages/publisher/src/generate-packages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import {
writeFile,
Logger,
cacheDir,
writeTgz,
} from "@definitelytyped/utils";
import {
getDefinitelyTyped,
Expand Down Expand Up @@ -63,7 +62,7 @@ export default async function generatePackages(
for (const { pkg, version } of changedPackages.changedTypings) {
await generateTypingPackage(pkg, allPackages, version, dt);
if (tgz) {
await writeTgz(outputDirectory(pkg), `${outputDirectory(pkg)}.tgz`);
await pacote.tarball.file(outputDirectory(pkg), `${outputDirectory(pkg)}.tgz`);
}
log(` * ${pkg.desc}`);
}
Expand Down
43 changes: 26 additions & 17 deletions packages/publisher/src/lib/package-publisher.ts
Original file line number Diff line number Diff line change
@@ -1,42 +1,51 @@
import assert = require("assert");
import { Logger, joinPaths, readFileAndWarn, NpmPublishClient } from "@definitelytyped/utils";
import { Logger, NpmPublishClient } from "@definitelytyped/utils";
import { NotNeededPackage, AnyPackage } from "@definitelytyped/definitions-parser";
import { updateTypeScriptVersionTags, updateLatestTag } from "@definitelytyped/retag";
import { updateTypeScriptVersionTags } from "@definitelytyped/retag";
import * as libpub from "libnpmpublish";
import * as pacote from "pacote";
import { ChangedTyping } from "./versions";
import { outputDirectory } from "../util/util";

// https://github.com/npm/types/pull/18
declare module "libnpmpublish" {
function publish(
manifest: Omit<import("@npm/types").PackageJson, "bundledDependencies">,
tarData: Buffer,
opts?: import("npm-registry-fetch").Options
): Promise<Response>;
}

export async function publishTypingsPackage(
client: NpmPublishClient,
changedTyping: ChangedTyping,
token: string,
dry: boolean,
log: Logger
): Promise<void> {
const { pkg, version, latestVersion } = changedTyping;
await common(client, pkg, log, dry);
const { pkg, version } = changedTyping;
await common(pkg, token, dry);
if (pkg.isLatest) {
await updateTypeScriptVersionTags(pkg, version, client, log, dry);
}
assert((latestVersion === undefined) === pkg.isLatest);
if (latestVersion !== undefined) {
// If this is an older version of the package, we still update tags for the *latest*.
// NPM will update "latest" even if we are publishing an older version of a package (https://github.com/npm/npm/issues/6778),
// so we must undo that by re-tagging latest.
await updateLatestTag(pkg.fullNpmName, latestVersion, client, log, dry);
}
}

export async function publishNotNeededPackage(
client: NpmPublishClient,
pkg: NotNeededPackage,
token: string,
dry: boolean,
log: Logger
): Promise<void> {
log(`Deprecating ${pkg.name}`);
await common(client, pkg, log, dry);
await common(pkg, token, dry);
}

async function common(client: NpmPublishClient, pkg: AnyPackage, log: Logger, dry: boolean): Promise<void> {
async function common(pkg: AnyPackage, token: string, dry: boolean): Promise<void> {
const packageDir = outputDirectory(pkg);
const packageJson = await readFileAndWarn("generate", joinPaths(packageDir, "package.json"));
await client.publish(packageDir, packageJson, dry, log);
const manifest = await pacote.manifest(packageDir).catch((reason) => {
throw reason.code === "ENOENT" ? new Error("Run generate first!", { cause: reason }) : reason;
});
const tarData = await pacote.tarball(packageDir);
// Make sure we never assign the latest tag to an old version.
if (!dry)
await libpub.publish(manifest, tarData, { defaultTag: pkg.isLatest ? "latest" : "", access: "public", token });
}
6 changes: 1 addition & 5 deletions packages/publisher/src/lib/versions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ export interface ChangedTyping {
readonly pkg: TypingsData;
/** This is the version to be published, meaning it's the version that doesn't exist yet. */
readonly version: string;
/** For a non-latest version, this is the latest version; publishing an old version updates the 'latest' tag and we want to change it back. */
readonly latestVersion: string | undefined;
}

export interface ChangedPackagesJson {
Expand All @@ -25,7 +23,6 @@ export interface ChangedPackagesJson {
export interface ChangedTypingJson {
readonly id: PackageId;
readonly version: string;
readonly latestVersion?: string;
}

export interface ChangedPackages {
Expand All @@ -37,10 +34,9 @@ export async function readChangedPackages(allPackages: AllPackages): Promise<Cha
const json = (await readDataFile("calculate-versions", versionsFilename)) as ChangedPackagesJson;
return {
changedTypings: json.changedTypings.map(
({ id, version, latestVersion }): ChangedTyping => ({
({ id, version }): ChangedTyping => ({
pkg: allPackages.getTypingsData(id),
version,
latestVersion,
})
),
changedNotNeededPackages: json.changedNotNeededPackages.map((id) =>
Expand Down
7 changes: 4 additions & 3 deletions packages/publisher/src/publish-packages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,13 @@ export default async function publishPackages(
log("=== Publishing packages ===");
}

const client = await NpmPublishClient.create(await getSecret(Secret.NPM_TOKEN), undefined);
const token = await getSecret(Secret.NPM_TOKEN);
const client = await NpmPublishClient.create(token, undefined);

for (const cp of changedPackages.changedTypings) {
log(`Publishing ${cp.pkg.desc}...`);

await publishTypingsPackage(client, cp, dry, log);
await publishTypingsPackage(client, cp, token, dry, log);

const commits = (await queryGithub(
`repos/DefinitelyTyped/DefinitelyTyped/commits?path=types%2f${cp.pkg.subDirectoryPath}`,
Expand Down Expand Up @@ -128,7 +129,7 @@ export default async function publishPackages(

for (const n of changedPackages.changedNotNeededPackages) {
const target = await skipBadPublishes(n, log);
await publishNotNeededPackage(client, target, dry, log);
await publishNotNeededPackage(target, token, dry, log);
}

await writeLog("publishing.md", logResult());
Expand Down
18 changes: 10 additions & 8 deletions packages/publisher/src/publish-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ import {
cacheDir,
isObject,
} from "@definitelytyped/utils";
import { PackageJson } from "@npm/types";
import * as libpub from "libnpmpublish";
import * as pacote from "pacote";
import * as semver from "semver";
// @ts-ignore
Expand Down Expand Up @@ -74,7 +76,7 @@ export default async function publishRegistry(dt: FS, allPackages: AllPackages,

const token = process.env.NPM_TOKEN!;

const publishClient = () => NpmPublishClient.create(token, { defaultTag: "next" });
const publishClient = () => NpmPublishClient.create(token);
if (maxVersion !== latestVersion) {
// There was an error in the last publish and types-registry wasn't validated.
// This may have just been due to a timeout, so test if types-registry@next is a subset of the one we're about to publish.
Expand All @@ -84,7 +86,7 @@ export default async function publishRegistry(dt: FS, allPackages: AllPackages,
await (await publishClient()).tag(typesRegistry, maxVersion, "latest", dry, log);
} else if (latestContentHash !== newContentHash) {
log("New packages have been added, so publishing a new registry.");
await publish(await publishClient(), typesRegistry, packageJson, newVersion, dry, log);
await publish(await publishClient(), packageJson, token, dry, log);
} else {
log("No new packages published, so no need to publish new registry.");
// Just making sure...
Expand Down Expand Up @@ -114,13 +116,13 @@ async function generate(registry: string, packageJson: {}): Promise<void> {

async function publish(
client: NpmPublishClient,
packageName: string,
packageJson: {},
version: string,
packageJson: PackageJson,
token: string,
dry: boolean,
log: Logger
): Promise<void> {
await client.publish(registryOutputPath, packageJson, dry, log);
const tarData = await pacote.tarball(registryOutputPath);
if (!dry) await libpub.publish(packageJson, tarData, { defaultTag: "next", access: "public", token });
// Sleep for 60 seconds to let NPM update.
if (dry) {
log("(dry) Skipping 60 second sleep...");
Expand All @@ -130,7 +132,7 @@ async function publish(
}
// Don't set it as "latest" until *after* it's been validated.
await validate(log);
await client.tag(packageName, version, "latest", dry, log);
await client.tag(packageJson.name, packageJson.version, "latest", dry, log);
}

async function installForValidate(log: Logger): Promise<void> {
Expand Down Expand Up @@ -205,7 +207,7 @@ function assertJsonNewer(newer: { [s: string]: any }, older: { [s: string]: any
}
}

function generatePackageJson(name: string, version: string, typesPublisherContentHash: string): object {
function generatePackageJson(name: string, version: string, typesPublisherContentHash: string): PackageJson {
const json = {
name,
version,
Expand Down
47 changes: 47 additions & 0 deletions packages/publisher/test/package-publisher.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { NotNeededPackage, TypingsData } from "@definitelytyped/definitions-parser";
import * as libpub from "libnpmpublish";
import { publishNotNeededPackage, publishTypingsPackage } from "../src/lib/package-publisher";

jest.mock("libnpmpublish");
jest.mock("pacote", () => ({ async manifest() {}, async tarball() {} }));

const client = { async tag() {} };

const latestVersion = {
pkg: new TypingsData({ typingsPackageName: "some-package" } as never, /*isLatest*/ true),
version: "1.2.3",
};
const oldVersion = {
pkg: new TypingsData({ typingsPackageName: "some-package" } as never, /*isLatest*/ false),
version: "1.2.3",
};
const notNeeded = new NotNeededPackage("not-needed", "not-needed", "1.2.3");

function log() {}

test("Latest version gets latest tag", async () => {
await publishTypingsPackage(client as never, latestVersion, /*token*/ "", /*dry*/ false, log);
expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
defaultTag: "latest",
access: "public",
token: "",
});
});

test("Publishing old versions doesn't change the latest tag", async () => {
await publishTypingsPackage(client as never, oldVersion, /*token*/ "", /*dry*/ false, log);
expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
defaultTag: "",
access: "public",
token: "",
});
});

test("Not-needed package gets latest tag", async () => {
await publishNotNeededPackage(notNeeded, /*token*/ "", /*dry*/ false, log);
expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
defaultTag: "latest",
access: "public",
token: "",
});
});
17 changes: 0 additions & 17 deletions packages/retag/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,11 @@ async function tag(dry: boolean, nProcesses: number, name?: string) {
const pkg = await AllPackages.readSingle(name);
const version = await getLatestTypingVersion(pkg);
await updateTypeScriptVersionTags(pkg, version, publishClient, consoleLogger.info, dry);
await updateLatestTag(pkg.fullNpmName, version, publishClient, consoleLogger.info, dry);
} else {
await nAtATime(10, await AllPackages.readLatestTypings(), async (pkg) => {
// Only update tags for the latest version of the package.
const version = await getLatestTypingVersion(pkg);
await updateTypeScriptVersionTags(pkg, version, publishClient, consoleLogger.info, dry);
await updateLatestTag(pkg.fullNpmName, version, publishClient, consoleLogger.info, dry);
});
}
// Don't tag notNeeded packages
Expand All @@ -94,21 +92,6 @@ export async function updateTypeScriptVersionTags(
}
}

export async function updateLatestTag(
fullName: string,
version: string,
client: NpmPublishClient,
log: Logger,
dry: boolean
): Promise<void> {
log(` but tag ${fullName}@${version} as "latest"`);
if (dry) {
log(' (dry) Skip move "latest" back to newest version');
} else {
await client.tag(fullName, version, "latest", dry, log);
}
}

export async function getLatestTypingVersion(pkg: TypingsData): Promise<string> {
return (await fetchTypesPackageVersionInfo(pkg, /*publish*/ false)).version;
}
Expand Down
1 change: 0 additions & 1 deletion packages/utils/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
logs
test/data/pack.tgz
1 change: 0 additions & 1 deletion packages/utils/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
"charm": "^1.0.2",
"fs-extra": "^8.1.0",
"fstream": "^1.0.12",
"tar": "^6.1.11",
"tar-stream": "^2.1.4"
},
"devDependencies": {
Expand Down
Loading

0 comments on commit 383193f

Please sign in to comment.