integration of protobuf to avoid memory issues (cf #42)

gildas-lormeau · Mar 15, 2020 · 318b252 · 318b252
1 parent 0977fec
commit 318b252
Show file tree

Hide file tree

Showing 7 changed files with 8,543 additions and 6 deletions.
diff --git a/extension/core/bg/business.js b/extension/core/bg/business.js
@@ -35,8 +35,10 @@ singlefile.extension.core.bg.business = (() => {
 		"common/index.js",
 		"common/ui/content/content-infobar.js",
 		"extension/lib/single-file/index.js",
+		"extension/lib/vendor/protobuf/protobuf.js",
 		"extension/core/index.js",
 		"extension/ui/index.js",
+		"extension/core/data/page-proto.js",
 		"extension/core/content/content-main.js",
 		"extension/core/content/content-download.js",
 		"extension/ui/content/content-ui-main.js"
@@ -152,7 +154,7 @@ singlefile.extension.core.bg.business = (() => {
 		const ui = singlefile.extension.ui.bg.main;
 		const tabs = singlefile.extension.core.bg.tabs;
 		const taskId = taskInfo.id;
-		return new Promise(async (resolve, reject) => {
+		return new Promise(async (resolve, reject) => { // eslint-disable-line no-async-promise-executor
 			taskInfo.status = "processing";
 			taskInfo.resolve = () => {
 				tasks.splice(tasks.findIndex(taskInfo => taskInfo.id == taskId), 1);

diff --git a/extension/core/bg/downloads.js b/extension/core/bg/downloads.js
@@ -21,7 +21,7 @@
  *   Source.
  */
 
-/* global browser, singlefile, URL, Response, GDrive */
+/* global browser, singlefile, URL, Response, GDrive, protobuf */
 
 singlefile.extension.core.bg.downloads = (() => {
 
@@ -96,7 +96,7 @@ singlefile.extension.core.bg.downloads = (() => {
 			contents = [message.content];
 		}
 		if (!message.truncated || message.finished) {
-			const pageData = JSON.parse(contents.join(""));
+			const pageData = protobuf.roots.default.Page.decode(contents.flat());
 			const blob = await singlefile.extension.core.bg.compression.compressPage(pageData, { insertTextBody: message.insertTextBody, url: tab.url });
 			await downloadBlob(blob, tab.id, tab.incognito, message);
 		}

diff --git a/extension/core/content/content-download.js b/extension/core/content/content-download.js
@@ -21,7 +21,7 @@
  *   Source.
  */
 
-/* global browser */
+/* global browser, protobuf */
 
 this.singlefile.extension.core.content.download = this.singlefile.extension.core.content.download || (() => {
 
@@ -33,7 +33,7 @@ this.singlefile.extension.core.content.download = this.singlefile.extension.core
 		if (options.includeBOM) {
 			pageData.content = "\ufeff" + pageData.content;
 		}
-		const content = JSON.stringify({ resources: pageData.resources, content: pageData.content, title: pageData.title, viewport: pageData.viewport, doctype: pageData.doctype });
+		const content = Array.from(protobuf.roots.default.Page.encode(pageData).finish());
 		for (let blockIndex = 0; blockIndex * MAX_CONTENT_SIZE < content.length; blockIndex++) {
 			const message = {
 				method: "downloads.download",
@@ -54,7 +54,7 @@ this.singlefile.extension.core.content.download = this.singlefile.extension.core
 			message.truncated = content.length > MAX_CONTENT_SIZE;
 			if (message.truncated) {
 				message.finished = (blockIndex + 1) * MAX_CONTENT_SIZE > content.length;
-				message.content = content.substring(blockIndex * MAX_CONTENT_SIZE, (blockIndex + 1) * MAX_CONTENT_SIZE);
+				message.content = content.slice(blockIndex * MAX_CONTENT_SIZE, (blockIndex + 1) * MAX_CONTENT_SIZE);
 			} else {
 				message.content = content;
 			}