0.4.0 - IDB Queue + Feats (#177)

* [feat] replace local storage with IndexedDB as queue backing

* [feat] add new integration system
    * adds arbitrary third party support

* [chore] move soupcan to integrations

* [chore] run prettier on shared.ts

* [fix] optionally chain refId to fix rare error

* [feat] add interception of requests on all following/followers pages
    * S/O @MrAwesome for suggesting/implementing this
    * Add logic to tag users intercepted from BlueVerifiedFollowers

---------

Co-authored-by: dani <[email protected]>
Co-authored-by: Eric Gallager <[email protected]>
Co-authored-by: Rouge <[email protected]>

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "blue-blocker",
-	"version": "0.3.9",
+	"version": "0.4.0",
 	"author": "DanielleMiu",
 	"description": "Blocks all Twitter Blue verified users on twitter.com",
 	"type": "module",

src/background/db.ts

@@ -1,5 +1,5 @@
-import { api, logstr, EventKey, LegacyVerifiedUrl, MessageEvent, ErrorEvent, HistoryStateUnblocked } from "../constants";
-import { commafy } from "../utilities";
+import { api, logstr, EventKey, LegacyVerifiedUrl, MessageEvent, ErrorEvent, HistoryStateBlocked, HistoryStateUnblocked } from "../constants";
+import { commafy, QueueId } from "../utilities";
 
 const expectedVerifiedUsersCount = 407520;
 let legacyDb: IDBDatabase;
@@ -29,8 +29,8 @@ export async function PopulateVerifiedDb() {
 		console.error(logstr, "failed to open legacy verified user database:", DBOpenRequest);
 	};
 
-	DBOpenRequest.onupgradeneeded = () => {
-		console.debug(logstr, "legacy db onupgradeneeded:", DBOpenRequest);
+	DBOpenRequest.onupgradeneeded = e => {
+		console.debug(logstr, "legacy db onupgradeneeded:", e);
 		legacyDb = DBOpenRequest.result;
 		if (legacyDb.objectStoreNames.contains(legacyDbStore)) {
 			return;
@@ -189,7 +189,8 @@ let db: IDBDatabase;
 
 const dbName = "blue-blocker-db";
 export const historyDbStore = "blocked_users";
-const dbVersion = 1;
+export const queueDbStore = "block_queue";
+const dbVersion = 2;
 // used so we don't load the db twice
 let dbLoaded: boolean = false;
 
@@ -203,8 +204,8 @@ export function ConnectDb(): Promise<IDBDatabase> {
 			return reject();
 		};
 
-		DBOpenRequest.onupgradeneeded = () => {
-			console.debug(logstr, "upgrading db:", DBOpenRequest);
+		DBOpenRequest.onupgradeneeded = e => {
+			console.debug(logstr, "upgrading db:", e);
 			db = DBOpenRequest.result;
 
 			if (!db.objectStoreNames.contains(historyDbStore)) {
@@ -214,6 +215,13 @@ export function ConnectDb(): Promise<IDBDatabase> {
 				store.createIndex("time", "time", { unique: false });
 				console.log(logstr, "created history database.");
 			}
+
+			if (!db.objectStoreNames.contains(queueDbStore)) {
+				const store = db.createObjectStore(queueDbStore, { keyPath: "user_id" });
+				store.createIndex("user_id", "user_id", { unique: true });
+				store.createIndex("queue", "queue", { unique: false });
+				console.log(logstr, "created queue database.");
+			}
 		};
 
 		DBOpenRequest.onsuccess = async () => {
@@ -223,6 +231,41 @@ export function ConnectDb(): Promise<IDBDatabase> {
 			}
 			dbLoaded = true;
 
+			const items = await api.storage.local.get({ BlockQueue: [] });
+			if (items?.BlockQueue?.length !== undefined && items?.BlockQueue?.length > 0) {
+				const transaction = db.transaction([queueDbStore], "readwrite");
+				transaction.onabort = transaction.onerror = reject;	
+				const store = transaction.objectStore(queueDbStore);
+
+				items.BlockQueue.forEach((item: BlockUser) => {
+					// required for users enqueued before 0.3.0
+					if (item.user.hasOwnProperty("legacy")) {
+						// @ts-ignore
+						item.user.name = item.user.legacy.name;
+						// @ts-ignore
+						item.user.screen_name = item.user.legacy.screen_name;
+						// @ts-ignore
+						delete item.user.legacy;
+					}
+
+					const user: QueueUser = {
+						user_id: item.user_id,
+						user: {
+							name: item.user.name,
+							screen_name: item.user.screen_name,
+						},
+						reason: item.reason,
+						queue: QueueId(),
+					};
+					// TODO: add error handling here
+					store.add(user);
+				});
+
+				api.storage.local.set({ BlockQueue: null });
+				transaction.commit();
+				console.debug(logstr, "imported", items.BlockQueue.length, "users from local storage queue");
+			}
+
 			console.log(logstr, "successfully connected to db");
 			return resolve(db);
 		};
@@ -285,3 +328,134 @@ export function RemoveUserFromHistory(user_id: string): Promise<void> {
 		})
 	);
 }
+
+interface QueueUser {
+	queue: number,
+	user_id: string,
+	user: { name: string, screen_name: string },
+	reason: number,
+	external_reason?: string,
+}
+
+export function AddUserToQueue(blockUser: BlockUser): Promise<void> {
+	const user: QueueUser = {
+		user_id: blockUser.user_id,
+		user: {
+			name: blockUser.user.name,
+			screen_name: blockUser.user.screen_name,
+		},
+		reason: blockUser.reason,
+		queue: QueueId(),
+	};
+
+	if (blockUser.external_reason) {
+		user.external_reason = blockUser.external_reason;
+	}
+
+	// @ts-ignore  // typescript is wrong here, this cannot return idb due to final throw
+	return new Promise<void>((resolve, reject) => {
+		const transaction = db.transaction([queueDbStore], "readwrite");
+		transaction.onabort = transaction.onerror = reject;
+
+		const store = transaction.objectStore(queueDbStore);
+		store.add(user);
+		transaction.oncomplete = () => resolve();
+		transaction.commit();
+	}).catch(e => {
+		if (e?.target?.error?.name !== "ConstraintError") {
+			// attempt to reconnect to the db
+			return ConnectDb()
+			.finally(() => {
+				throw e;  // re-throw error to retry
+			});
+		}
+	});
+}
+
+export function PopUserFromQueue(): Promise<BlockUser | null> {
+	// @ts-ignore  // typescript is wrong here, this cannot return idb due to final throw
+	return new Promise<BlockUser | null>(async (resolve, reject) => {
+		const transaction = db.transaction([queueDbStore], "readwrite");
+		transaction.onabort = transaction.onerror = reject;
+		const store = transaction.objectStore(queueDbStore);
+		const index = store.index("queue");
+
+		const result = await new Promise<QueueUser | undefined>((res, rej) => {
+			const req = index.get(IDBKeyRange.bound(Number.MIN_VALUE, Number.MAX_VALUE));
+			req.onerror = rej;
+			req.onsuccess = () => {
+				res(req.result as QueueUser);
+			};
+		});
+
+		if (result === undefined) {
+			return resolve(null);
+		}
+
+		const user: BlockUser = {
+			user_id: result.user_id,
+			user: {
+				name: result.user.name,
+				screen_name: result.user.screen_name,
+			},
+			reason: result.reason,
+		};
+
+		if (result.external_reason) {
+			user.external_reason = result.external_reason;
+		}
+
+		store.delete(user.user_id);
+		transaction.commit();
+		transaction.oncomplete = () => resolve(user);
+	}).catch(e =>
+		// attempt to reconnect to the db
+		ConnectDb()
+		.finally(() => {
+			throw e;  // re-throw error to retry
+		})
+	);
+}
+
+export function WholeQueue(): Promise<BlockUser[]> {
+	return ConnectDb().then(qdb => {
+		return new Promise<BlockUser[]>((resolve, reject) => {
+			const transaction = qdb.transaction([queueDbStore], "readonly");
+			transaction.onabort = transaction.onerror = reject;
+			const store = transaction.objectStore(queueDbStore);
+			const index = store.index("queue");
+			const req = index.getAll(IDBKeyRange.bound(Number.MIN_VALUE, Number.MAX_VALUE), 10000);
+
+			req.onerror = reject;
+			req.onsuccess = () => {
+				const users = req.result as BlockUser[];
+				resolve(users);
+			};
+		});
+	}).catch(() => 
+		api.storage.local.get({ BlockQueue: [] }).then(items =>
+			items?.BlockQueue
+		)
+	);
+}
+
+export function QueueLength(): Promise<number> {
+	return ConnectDb().then(qdb => {
+		return new Promise<number>((resolve, reject) => {
+			const transaction = qdb.transaction([queueDbStore], "readonly");
+			transaction.onabort = transaction.onerror = reject;
+			const store = transaction.objectStore(queueDbStore);
+			const req = store.count();
+
+			req.onerror = reject;
+			req.onsuccess = () => {
+				const users = req.result as number;
+				resolve(users);
+			};
+		});
+	}).catch(() => 
+		api.storage.local.get({ BlockQueue: [] }).then(items =>
+			items?.BlockQueue?.length
+		)
+	);
+}

src/background/index.ts

@@ -1,7 +1,6 @@
-import { api, logstr, AddToHistoryAction, ErrorStatus, IsVerifiedAction, ReasonExternal, RemoveFromHistoryAction, SoupcanExtensionId, SuccessStatus, DefaultOptions } from '../constants';
+import { api, logstr, AddToHistoryAction, ErrorStatus, IsVerifiedAction, ReasonExternal, RemoveFromHistoryAction, SoupcanExtensionId, SuccessStatus, DefaultOptions, AddToQueueAction, PopFromQueueAction, IntegrationStateSendAndReceive, IntegrationStateDisabled, IntegrationStateSendOnly } from '../constants';
 import { abbreviate, RefId } from '../utilities';
-import { AddUserToHistory, CheckDbIsUserLegacyVerified, ConnectDb, PopulateVerifiedDb, RemoveUserFromHistory } from './db';
-import { BlockQueue } from '../models/block_queue';
+import { AddUserToHistory, AddUserToQueue, CheckDbIsUserLegacyVerified, ConnectDb, PopUserFromQueue, PopulateVerifiedDb, RemoveUserFromHistory } from './db';
 
 api.action.setBadgeBackgroundColor({ color: "#666" });
 if (api.action.hasOwnProperty("setBadgeTextColor")) {
@@ -66,6 +65,18 @@ api.runtime.onMessage.addListener((m, s, r) => { let response: MessageResponse;
 				response = { status: SuccessStatus, result: null } as SuccessResponse;
 				break;
 
+			case AddToQueueAction:
+				const addToQueueMessage = message.data as BlockUser;
+				await AddUserToQueue(addToQueueMessage);
+				response = { status: SuccessStatus, result: null } as SuccessResponse;
+				break;
+
+			case PopFromQueueAction:
+				// no payload with this request
+				const user = await PopUserFromQueue();
+				response = { status: SuccessStatus, result: user } as SuccessResponse;
+				break;
+
 			default:
 				console.error(logstr, refid, "got a message that couldn't be handled from sender:", sender, message);
 				response = { status: ErrorStatus, message: "unknown action" } as ErrorResponse;
@@ -80,14 +91,19 @@ api.runtime.onMessage.addListener((m, s, r) => { let response: MessageResponse;
 
 ////////////////////////////////////////////////// EXTERNAL MESSAGE HANDLING //////////////////////////////////////////////////
 
-const queue = new BlockQueue(api.storage.local);
-const [blockAction] = ["BLOCK"];
-const allowedExtensionIds = new Set([SoupcanExtensionId]);
+const [blockActionV1, blockAction] = ["BLOCK", "block_user"];
 
 api.runtime.onMessageExternal.addListener((m, s, r) => { let response: MessageResponse; (async (message, sender) => {
 	const refid = RefId();
 	console.debug(logstr, refid, "ext recv:", message, sender);
-	if (!allowedExtensionIds.has(sender?.id ?? "")) {
+	const integrations = await api.storage.local.get({ soupcanIntegration: false, integrations: { } }) as { [id: string]: Integration };
+	const senderId = sender.id ?? "";
+	if (!integrations.hasOwnProperty(senderId)) {
+		response = { status: ErrorStatus, message: "extension not allowed" } as ErrorResponse;
+		return;
+	}
+	if (integrations[senderId].state === IntegrationStateDisabled || integrations[senderId].state === IntegrationStateSendOnly) {
+		response = { status: ErrorStatus, message: "extension disabled or not allowed to send messages" } as ErrorResponse;
 		return;
 	}
 
@@ -97,10 +113,16 @@ api.runtime.onMessageExternal.addListener((m, s, r) => { let response: MessageRe
 	// other message contents change based on the defined action
 	try {
 		switch (message?.action) {
+			case blockActionV1:
+				const blockV1Message = message as { user_id: string, name: string, screen_name: string, reason: string };
+				const userV1: BlockUser = { user_id: blockV1Message.user_id, user: { name: blockV1Message.name, screen_name: blockV1Message.screen_name }, reason: ReasonExternal, external_reason: blockV1Message.reason };
+				await AddUserToQueue(userV1).catch(() => AddUserToQueue(userV1));
+				response = { status: SuccessStatus, result: "user queued for blocking" } as SuccessResponse;
+				break;
+
 			case blockAction:
-				const blockMessage = message as { user_id: string, name: string, screen_name: string, reason: string };
-				const user: BlockUser = { user_id: blockMessage.user_id, user: { name: blockMessage.name, screen_name: blockMessage.screen_name }, reason: ReasonExternal, external_reason: blockMessage.reason };
-				await queue.push(user);
+				const blockMessage = message.data as BlockUser;
+				await AddUserToQueue(blockMessage).catch(() => AddUserToQueue(blockMessage));
 				response = { status: SuccessStatus, result: "user queued for blocking" } as SuccessResponse;
 				break;
 
@@ -113,5 +135,5 @@ api.runtime.onMessageExternal.addListener((m, s, r) => { let response: MessageRe
 		console.error(logstr, refid, "unexpected error caught during", message?.action, "action", e);
 		response = { status: ErrorStatus, message: e.message ?? "unknown error" } as ErrorResponse;
 	}
-	console.debug(logstr, refid, "respond:", response);
+	console.debug(logstr, refid, "ext respond:", response);
 })(m, s).finally(() => r(response)); return true });

src/constants.ts

@@ -83,6 +83,7 @@ export const ReasonMap = {
 	[ReasonTransphobia]: 'transphobia',
 	[ReasonPromoted]: 'promoting tweets',
 };
+
 export const LegacyVerifiedUrl: string =
 	'https://gist.githubusercontent.com/travisbrown/b50d6745298cccd6b1f4697e4ec22103/raw/012009351630dc351e3a763b49bf24fa50ca3eb7/legacy-verified.csv';
 export const Browser =
@@ -93,10 +94,11 @@ export const SoupcanExtensionId =
 	Browser === 'chrome' ? 'hcneafegcikghlbibfmlgadahjfckonj' : '[email protected]';
 
 // internal message actions
-export const [IsVerifiedAction, AddToHistoryAction, RemoveFromHistoryAction] = [
+export const [IsVerifiedAction, AddToHistoryAction, RemoveFromHistoryAction, AddToQueueAction, PopFromQueueAction] = [
 	'is_verified',
 	'add_user_to_history',
 	'remove_user_from_history',
+  'add_user_to_queue', 'pop_user_from_queue'
 ];
 export const SuccessStatus: SuccessStatus = 'SUCCESS';
 export const ErrorStatus: ErrorStatus = 'ERROR';
@@ -105,3 +107,8 @@ export const ErrorStatus: ErrorStatus = 'ERROR';
 export const EventKey = 'MultiTabEvent';
 export const ErrorEvent = 'ErrorEvent';
 export const MessageEvent = 'MessageEvent';
+
+export const IntegrationStateDisabled = 0;
+export const IntegrationStateReceiveOnly= 1;
+export const IntegrationStateSendAndReceive = 2;
+export const IntegrationStateSendOnly = 3;

src/content/index.ts

@@ -34,6 +34,11 @@ document.addEventListener("blue-blocker-event", function (e: CustomEvent<BlueBlo
 				case "SearchTimeline":
 				case "UserTweets":
 				case "TweetDetail":
+				case "Following":
+				case "Followers":
+				case "UserCreatorSubscriptions":
+				case "FollowersYouKnow":
+				case "BlueVerifiedFollowers":
 					return HandleInstructionsResponse(e, parsed_body, config);
 				case "timeline/home.json":
 				case "search/adaptive.json":

src/global.d.ts

@@ -66,6 +66,11 @@ interface ErrorResponse {
 	message: string;
 }
 
+interface ExternalBlockResponse {
+	block: boolean,
+	reason?: string,
+}
+
 interface BlueBlockerEvent {
 	url: URL | string;
 	parsedUrl: RegExpExecArray;
@@ -110,3 +115,8 @@ interface BlockedUser {
 	state: number;
 	time: Date;
 }
+
+interface Integration {
+	name: string,
+	state: number,
+}

src/injected/inject.ts

@@ -2,7 +2,7 @@
 
 (function (xhr) {
 	// TODO: find a way to make this cleaner
-	const RequestRegex = /^https?:\/\/(?:\w+\.)?twitter.com\/[\w\/\.\-\_\=]+\/(HomeLatestTimeline|HomeTimeline|SearchTimeline|UserTweets|timeline\/home\.json|TweetDetail|search\/typeahead\.json|search\/adaptive\.json|blocks\/destroy\.json|mutes\/users\/destroy\.json)(?:$|\?)/;
+	const RequestRegex = /^https?:\/\/(?:\w+\.)?twitter.com\/[\w\/\.\-\_\=]+\/(HomeLatestTimeline|HomeTimeline|Followers|Following|SearchTimeline|UserTweets|UserCreatorSubscriptions|FollowersYouKnow|BlueVerifiedFollowers|SearchTimeline|timeline\/home\.json|TweetDetail|search\/typeahead\.json|search\/adaptive\.json|blocks\/destroy\.json|mutes\/users\/destroy\.json)(?:$|\?)/;
 
 	let XHR = <BlueBlockerXLMRequest>XMLHttpRequest.prototype;
 	let open = XHR.open;

src/manifest.ts

@@ -3,7 +3,7 @@ import { defineManifest } from "@crxjs/vite-plugin";
 export default defineManifest({
 	name: "Blue Blocker",
 	description: "Blocks all Twitter Blue verified users on twitter.com",
-	version: "0.3.9",
+	version: "0.4.0",
 	manifest_version: 3,
 	icons: {
 		"128": "icon/icon-128.png",