Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix compression backpressure on large files #155

Merged
merged 1 commit into from
Oct 12, 2020
Merged

Fix compression backpressure on large files #155

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 31 additions & 25 deletions src/lib/polkompress.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,28 @@
/**
* To-Be-Published
* https://gist.github.com/developit/1cc31d2ce6f3374e2d21941f27788ecc
*/

import zlib from 'zlib';

function getChunkSize(chunk, enc) {
if (!chunk) return 0;
return Buffer.byteLength(chunk, enc);
}
const MIMES = /text|javascript|\/json|xml/i;

const noop = () => {};

const MIMES = /text|javascript|\/json|xml/i;
const getChunkSize = (chunk, enc) => (chunk ? Buffer.byteLength(chunk, enc) : 0);

/** @returns {import('polka').Middleware} */
export default function compress({ threshold = 1024, level = -1, brotli = false, gzip = true, mimes = MIMES } = {}) {
/**
* @param {object} [options]
* @param {number} [options.threshold = 1024] Don't compress responses below this size (in bytes)
* @param {number} [options.level = -1] Gzip/Brotli compression effort (1-11, or -1 for default)
* @param {boolean} [options.brotli = false] Generate and serve Brotli-compressed responses
* @param {boolean} [options.gzip = true] Generate and serve Gzip-compressed responses
* @param {RegExp} [options.mimes] Regular expression of response MIME types to compress (default: text|javascript|json|xml)
* @returns {(req: Pick<import('http').IncomingMessage, 'method'|'headers'>, res: import('http').ServerResponse, next?:Function) => void}
* @retur {import('polka').Middleware}
*/
export default function compression({ threshold = 1024, level = -1, brotli = false, gzip = true, mimes = MIMES } = {}) {
const brotliOpts = (typeof brotli === 'object' && brotli) || {};
const gzipOpts = (typeof gzip === 'object' && gzip) || {};

// disable Brotli on Node<12.7 where it is unsupported:
if (!zlib.createBrotliCompress) brotli = false;

return (req, res, next = noop) => {
const accept = req.headers['accept-encoding'] + '';
const encoding = ((brotli && accept.match(/\bbr\b/)) || (gzip && accept.match(/\bgzip\b/)) || [])[0];
Expand All @@ -29,15 +33,18 @@ export default function compress({ threshold = 1024, level = -1, brotli = false,
/** @type {zlib.Gzip | zlib.BrotliCompress} */
let compress;
let pendingStatus;
/** @type {[string, function][]?} */
let pendingListeners = [];
let started = false;
let size = 0;

function start() {
started = true;
// @ts-ignore
size = res.getHeader('Content-Length') | 0 || size;
const compressible = mimes.test(res.getHeader('Content-Type') + '');
const compressible = mimes.test(String(res.getHeader('Content-Type') || 'text/plain'));
const cleartext = !res.getHeader('Content-Encoding');
const listeners = pendingListeners || [];
if (compressible && cleartext && size >= threshold) {
res.setHeader('Content-Encoding', encoding);
res.removeHeader('Content-Length');
Expand All @@ -51,17 +58,15 @@ export default function compress({ threshold = 1024, level = -1, brotli = false,
compress = zlib.createGzip(Object.assign({ level }, gzipOpts));
}
// backpressure
compress.on('data', (...args) => write.apply(res, args) === false && compress.pause());
compress.on('data', chunk => write.call(res, chunk) === false && compress.pause());
on.call(res, 'drain', () => compress.resume());
compress.on('end', (...args) => end.apply(res, args));
// const start = Date.now();
// compress.on('end', () => {
// console.log(`${req.url}[${(size / 1000) | 0}kb]: ${encoding};dur=${Date.now() - start}`);
// });
compress.on('end', () => end.call(res));
listeners.forEach(p => compress.on.apply(compress, p));
} else {
pendingListeners = null;
listeners.forEach(p => on.apply(res, p));
}
const listeners = pendingListeners;
pendingListeners = null;
listeners.forEach(p => on.apply(res, p));

if (pendingStatus) writeHead.call(res, pendingStatus);
}

Expand All @@ -73,12 +78,14 @@ export default function compress({ threshold = 1024, level = -1, brotli = false,
pendingStatus = status;
return this;
};

res.write = function (chunk, enc, cb) {
size += getChunkSize(chunk, enc);
if (!started) start();
if (!compress) return write.apply(this, arguments);
return compress.write.apply(compress, arguments);
};

res.end = function (chunk, enc, cb) {
if (arguments.length > 0 && typeof chunk !== 'function') {
size += getChunkSize(chunk, enc);
Expand All @@ -87,10 +94,9 @@ export default function compress({ threshold = 1024, level = -1, brotli = false,
if (!compress) return end.apply(this, arguments);
return compress.end.apply(compress, arguments);
};
/** Not currently used. */
let pendingListeners = [];

res.on = function (type, listener) {
if (!pendingListeners) on.call(this, type, listener);
if (!pendingListeners || type !== 'drain') on.call(this, type, listener);
else if (compress) compress.on(type, listener);
else pendingListeners.push([type, listener]);
return this;
Expand Down