diff --git a/package.json b/package.json index d7e0ba8..380bd12 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "defillama-extension", "private": true, - "version": "0.0.8.2", + "version": "0.0.8.3", "type": "module", "description": "DefiLlama Extension", "displayName": "DefiLlama", @@ -18,8 +18,6 @@ "@emotion/react": "^11", "@emotion/styled": "^11", "@tanstack/react-query": "^4.8.0", - "dexie": "^3.2.2", - "dexie-react-hooks": "^1.1.1", "fast-levenshtein": "^3.0.0", "framer-motion": "^7.5.1", "react": "^18.2.0", diff --git a/src/pages/background/index.ts b/src/pages/background/index.ts index f679a59..ef12f03 100644 --- a/src/pages/background/index.ts +++ b/src/pages/background/index.ts @@ -7,18 +7,9 @@ import maxPain from "@assets/img/memes/max-pain-128.png"; import que from "@assets/img/memes/que-128.png"; import upOnly from "@assets/img/memes/up-only-128.png"; -import { Protocol, protocolsDb, allowedDomainsDb, blockedDomainsDb, fuzzyDomainsDb } from "../libs/db"; -import { - PROTOCOLS_API, - METAMASK_LIST_CONFIG_API, - DEFILLAMA_DIRECTORY_API, - PROTOCOL_TVL_THRESHOLD, -} from "../libs/constants"; import { getStorage } from "../libs/helpers"; import { checkDomain } from "../libs/phishing-detector"; -startupTasks(); - async function getCurrentTab() { const queryOptions = { active: true, currentWindow: true }; const [tab] = await Browser.tabs.query(queryOptions); @@ -37,6 +28,10 @@ async function handlePhishingCheck() { let reason = "Unknown website"; const tab = await getCurrentTab(); try { + if (!tab) { + console.log('Unable to get current tab'); + return; + } const url = tab.url; if (url.startsWith("https://metamask.github.io/phishing-warning")) { // already captured and redirected to metamask phishing warning page @@ -90,95 +85,6 @@ async function handlePhishingCheck() { } } -export async function updateProtocolsDb() { - const raw = await fetch(PROTOCOLS_API).then((res) => res.json()); - const protocols = (raw["protocols"]?.map((x: any) => ({ - name: x.name, - url: x.url, - logo: x.logo, - category: x.category, - tvl: x.tvl, - })) ?? []) as Protocol[]; - if (protocols.length === 0) { - console.log("updateProtocolsDb", "no protocols found"); - return; - } - // empty db before updating - await protocolsDb.protocols.clear(); - const result = await protocolsDb.protocols.bulkPut(protocols); - console.log("updateProtocolsDb", result); -} - -export async function updateDomainDbs() { - console.log("updateDomainDbs", "start"); - const rawProtocols = await fetch(PROTOCOLS_API).then((res) => res.json()); - const protocols = ( - (rawProtocols["protocols"]?.map((x: any) => ({ - name: x.name, - url: x.url, - logo: x.logo, - category: x.category, - tvl: x.tvl || 0, - })) ?? []) as Protocol[] - ).filter((x) => x.tvl >= PROTOCOL_TVL_THRESHOLD); - const protocolDomains = protocols - .map((x) => { - try { - return new URL(x.url).hostname.replace("www.", ""); - } catch (error) { - console.log("updateDomainDbs", "error", error); - return null; - } - }) - .filter((x) => x !== null) - .map((x) => ({ domain: x })); - const metamaskLists = (await fetch(METAMASK_LIST_CONFIG_API).then((res) => res.json())) as { - fuzzylist: string[]; - whitelist: string[]; - blacklist: string[]; - }; - const metamaskFuzzyDomains = metamaskLists.fuzzylist.map((x) => ({ domain: x })); - const metamaskAllowedDomains = metamaskLists.whitelist.map((x) => ({ domain: x })); - const metamaskBlockedDomains = metamaskLists.blacklist.map((x) => ({ domain: x })); - const rawDefillamaDirectory = (await fetch(DEFILLAMA_DIRECTORY_API).then((res) => res.json())) as { - version: number; - whitelist: string[]; - blacklist?: string[]; - fuzzylist?: string[]; - }; - const defillamaDomains = rawDefillamaDirectory.whitelist.map((x) => ({ domain: x })); - const defillamaBlockedDomains = rawDefillamaDirectory.blacklist?.map((x) => ({ domain: x })) ?? []; - const defillamaFuzzyDomains = rawDefillamaDirectory.fuzzylist?.map((x) => ({ domain: x })) ?? []; - const allowedDomains = [metamaskAllowedDomains, protocolDomains, defillamaDomains].flat(); - if (allowedDomains.length === 0) { - console.log("allowedDomainsDb", "no allowed domains fetched, skipping update"); - } else { - allowedDomainsDb.domains.clear(); - allowedDomainsDb.domains.bulkPut(allowedDomains); - console.log("allowedDomainsDb", await allowedDomainsDb.domains.count()); - } - - const blockedDomains = [metamaskBlockedDomains, defillamaBlockedDomains].flat(); - if (blockedDomains.length === 0) { - console.log("blockedDomainsDb", "no blocked domains fetched, skipping update"); - } else { - blockedDomainsDb.domains.clear(); - blockedDomainsDb.domains.bulkPut(blockedDomains); - console.log("blockedDomainsDb", await blockedDomainsDb.domains.count()); - } - - const fuzzyDomains = [metamaskFuzzyDomains, protocolDomains, defillamaDomains, defillamaFuzzyDomains].flat(); - if (fuzzyDomains.length === 0) { - console.log("fuzzyDomainsDb", "no fuzzy domains fetched, skipping update"); - } else { - fuzzyDomainsDb.domains.clear(); - fuzzyDomainsDb.domains.bulkPut(fuzzyDomains); - console.log("fuzzyDomainsDb", await fuzzyDomainsDb.domains.count()); - } - - console.log("updateDomainDbs", "done"); -} - // monitor updates to the tab, specifically when the user navigates to a new page (new url) Browser.tabs.onUpdated.addListener(async (tabId, onUpdatedInfo, tab) => { // console.log("onUpdated", onUpdatedInfo.status, onUpdatedInfo.url); @@ -194,48 +100,3 @@ Browser.tabs.onActivated.addListener(async (onActivatedInfo) => { await Browser.tabs.sendMessage(onActivatedInfo.tabId, { message: "TabActivated" }); await handlePhishingCheck(); }); - -async function setupUpdateProtocolsDb() { - console.log("setupUpdateProtocolsDb"); - await Browser.alarms.clear("updateProtocolsDb"); - - console.log("setupUpdateProtocolsDb", "create"); - await updateProtocolsDb(); - Browser.alarms.create("updateProtocolsDb", { periodInMinutes: 4 * 60 }); // update once every 4 hours -} - -async function setupUpdateDomainDbs() { - console.log("setupUpdateDomainDbs"); - await Browser.alarms.clear("updateDomainDbs"); - - console.log("setupUpdateDomainDbs", "create"); - await updateDomainDbs(); - Browser.alarms.create("updateDomainDbs", { periodInMinutes: 4 * 60 }); // update once every 4 hours -} - -function startupTasks() { - console.log("startupTasks", "start"); - setupUpdateProtocolsDb(); - setupUpdateDomainDbs(); - Browser.action.setIcon({ path: cute }); - console.log("startupTasks", "done"); -} - -Browser.runtime.onInstalled.addListener(() => { - startupTasks(); -}); - -Browser.runtime.onStartup.addListener(() => { - startupTasks(); -}); - -Browser.alarms.onAlarm.addListener(async (a) => { - switch (a.name) { - case "updateProtocolsDb": - await updateProtocolsDb(); - break; - case "updateDomainDbs": - await updateDomainDbs(); - break; - } -}); diff --git a/src/pages/content/components/twitter/pageHandlers.tsx b/src/pages/content/components/twitter/pageHandlers.tsx index 705de05..8baa11e 100644 --- a/src/pages/content/components/twitter/pageHandlers.tsx +++ b/src/pages/content/components/twitter/pageHandlers.tsx @@ -107,6 +107,7 @@ export async function handleTweetStatusPage({ twitterCashTags, twitterHashTags, } else { handleAdTweet(tweet); + /* disabling this as scammers stopped using it // if the tweet text content consists of only numbers, then it's sus. Add red background the tweet const onlyNumbers = /^[0-9]+$/.test(tweetText) // it is not number but probably gibberish word @@ -115,6 +116,7 @@ export async function handleTweetStatusPage({ twitterCashTags, twitterHashTags, handleSusTweet(tweet, isLinkedTweet, "onlyNumbers", "BG_RED"); return; } + */ // only hide addresses if not from the op if (!!tweetText) handleTweetWithAddress(tweet, tweetText, isLinkedTweet); diff --git a/src/pages/libs/db.ts b/src/pages/libs/db.ts index a806d25..bea1fad 100644 --- a/src/pages/libs/db.ts +++ b/src/pages/libs/db.ts @@ -1,65 +1,134 @@ -import Dexie, { Table } from "dexie"; +import { fetchData } from './storage' +import { version } from '../../../package.json' +import Browser from "webextension-polyfill"; +import cute from "@assets/img/memes/cute-128.png"; + +import { + PROTOCOLS_API, + METAMASK_LIST_CONFIG_API, + DEFILLAMA_DIRECTORY_API, + PROTOCOL_TVL_THRESHOLD, +} from "./constants"; export interface Protocol { - name: string; url: string; - logo: string; - category: string; tvl?: number; } -export class ProtocolsDb extends Dexie { - protocols!: Table; - - constructor() { - super("ProtocolsDb"); - this.version(1).stores({ - protocols: "name, category", - }); - } +export const blockedDomainsDb: { + data: Set +} = { + data: new Set() } -export const protocolsDb = new ProtocolsDb(); +export const fuzzyDomainsDb: { + data: string[] +} = { + data: [] +} -export interface Domain { - domain: string; +export const allowedDomainsDb: { + data: Set +} = { + data: new Set() } -export class AllowedDomainsDb extends Dexie { - domains!: Table; +const cacheKey = 'cache-v' + version - constructor() { - super("AllowedDomainsDb"); - this.version(1).stores({ - domains: "domain", - }); - } -} +async function getData() { + console.time(cacheKey) + const rawProtocols = await fetch(PROTOCOLS_API).then((res) => res.json()); + const protocols = ( + (rawProtocols["protocols"]?.map((x: any) => ({ + url: x.url, + tvl: x.tvl || 0, + })) ?? []) as Protocol[] + ).filter((x) => x.tvl >= PROTOCOL_TVL_THRESHOLD); + const protocolDomains = protocols + .map((x) => { + try { + if (!x.url) return null; + return new URL(x.url).hostname.replace("www.", ""); + } catch (error) { + console.log("updateDomainDbs", "error", error); + return null; + } + }) + .filter((x) => x !== null) + console.log("updateDomainDbs", "protocolDomains", protocolDomains.length); + const metamaskLists = (await fetch(METAMASK_LIST_CONFIG_API).then((res) => res.json())) as { + fuzzylist: string[]; + whitelist: string[]; + blacklist: string[]; + }; + const metamaskFuzzyDomains = metamaskLists.fuzzylist; + const metamaskAllowedDomains = metamaskLists.whitelist; + const metamaskBlockedDomains = metamaskLists.blacklist; + const rawDefillamaDirectory = (await fetch(DEFILLAMA_DIRECTORY_API).then((res) => res.json())) as { + version: number; + whitelist: string[]; + blacklist?: string[]; + fuzzylist?: string[]; + }; + const defillamaDomains = rawDefillamaDirectory.whitelist; + const defillamaBlockedDomains = rawDefillamaDirectory.blacklist ?? []; + const defillamaFuzzyDomains = rawDefillamaDirectory.fuzzylist ?? []; + const allowedDomains = getUniqueItems(metamaskAllowedDomains, protocolDomains, defillamaDomains, ['x.com']) + console.log("allowedDomainsDb", allowedDomains.length); -export const allowedDomainsDb = new AllowedDomainsDb(); + const blockedDomains = getUniqueItems(metamaskBlockedDomains, defillamaBlockedDomains) + console.log("blockedDomainsDb", blockedDomains.length); -export class FuzzyDomainsDb extends Dexie { - domains!: Table; + const fuzzyDomains = getUniqueItems(metamaskFuzzyDomains, protocolDomains, defillamaDomains, defillamaFuzzyDomains) + console.log("fuzzyDomainsDb", await fuzzyDomains.length); - constructor() { - super("FuzzyDomainsDb"); - this.version(1).stores({ - domains: "domain", - }); + console.timeEnd(cacheKey) + return { + allowedDomains, + blockedDomains, + fuzzyDomains, } } -export const fuzzyDomainsDb = new FuzzyDomainsDb(); +function getUniqueItems(...arrays) { + const allItems = arrays.flat() + return [...new Set(allItems)] -export class BlockedDomainsDb extends Dexie { - domains!: Table; +} + +async function updateDb() { + const { + allowedDomains = [], + blockedDomains = [], + fuzzyDomains = [], + } = await fetchData({ + key: cacheKey, + updateFrequency: 60 * 60 * 4, // update every 4 hours + getData, + }) + allowedDomainsDb.data = new Set(allowedDomains) + blockedDomainsDb.data = new Set(blockedDomains) + fuzzyDomainsDb.data = fuzzyDomains +} - constructor() { - super("BlockedDomainsDb"); - this.version(1).stores({ - domains: "domain", - }); +updateDb() +// setInterval(updateDb, 1000 * 6 * 10) // run every 10 minutes +Browser.alarms.create("updateDomainDbs", { periodInMinutes: 6 * 60 }); // update every 6 hours + +Browser.alarms.onAlarm.addListener(async (a) => { + switch (a.name) { + case "updateDomainDbs": + await updateDb(); + break; } +}) + +async function startupTasks() { + console.time("startupTasks"); + await updateDb(); + Browser.action.setIcon({ path: cute }); + console.timeEnd("startupTasks"); } -export const blockedDomainsDb = new BlockedDomainsDb(); +Browser.runtime.onInstalled.addListener(startupTasks) +Browser.runtime.onStartup.addListener(startupTasks) \ No newline at end of file diff --git a/src/pages/libs/hooks.ts b/src/pages/libs/hooks.ts index 3971182..5ab5c9a 100644 --- a/src/pages/libs/hooks.ts +++ b/src/pages/libs/hooks.ts @@ -1,18 +1,6 @@ import { useCallback, useEffect, useState } from "react"; -import { useLiveQuery } from "dexie-react-hooks"; -import { Protocol, protocolsDb } from "./db"; import Browser from "webextension-polyfill"; -/** - * Protocols data synced with IndexedDB and updated every 4 hours using Dexie. - * - * @returns {Protocol[]} protocols - */ -export const useProtocols = (): Protocol[] => - useLiveQuery(async () => { - return await protocolsDb.protocols.toArray(); - }); - /** * State synced with local storage. Updates itself when local storage changes based on event listener. * diff --git a/src/pages/libs/phishing-detector.ts b/src/pages/libs/phishing-detector.ts index 4aba3a9..a512f35 100644 --- a/src/pages/libs/phishing-detector.ts +++ b/src/pages/libs/phishing-detector.ts @@ -9,7 +9,7 @@ interface CheckDomainResult { extra?: string; } -let domainCheckCache = new Map>(); +let domainCheckCache = new Map(); let lastCacheClear = Date.now(); export function clearDomainCheckCache() { @@ -22,47 +22,39 @@ export function clearDomainCheckCache() { export async function checkDomain(domain: string): Promise { clearDomainCheckCache(); - if (!domainCheckCache.has(domain)) + if (!domainCheckCache.has(domain)) { domainCheckCache.set(domain, _checkDomain(domain)); + } + return domainCheckCache.get(domain); } -async function _checkDomain(domain: string): Promise { - console.log("Checking domain", domain); +function _checkDomain(domain: string): CheckDomainResult { const topLevelDomain = domain.split(".").slice(-2).join("."); - const isAllowed = - (await allowedDomainsDb.domains.get({ domain })) || - (await allowedDomainsDb.domains.get({ domain: topLevelDomain })); - if (isAllowed) { + const isAllowed = allowedDomainsDb.data.has(domain) || allowedDomainsDb.data.has(topLevelDomain); + if (isAllowed) return { result: false, type: "allowed" }; - } - const isBlocked = - (await blockedDomainsDb.domains.get({ domain })) || - (await blockedDomainsDb.domains.get({ domain: topLevelDomain })); - if (isBlocked) { + + const isBlocked = blockedDomainsDb.data.has(domain) || blockedDomainsDb.data.has(topLevelDomain); + if (isBlocked) return { result: true, type: "blocked" }; - } - const fuzzyDomains = fuzzyDomainsDb.domains.toCollection(); - let fuzzyResult: CheckDomainResult; - await fuzzyDomains - .until((x) => { - const distance = levenshtein.get(x.domain, domain); - const distanceTop = levenshtein.get(x.domain, topLevelDomain); - const isMatched = distance <= DEFAULT_LEVENSHTEIN_TOLERANCE || distanceTop <= DEFAULT_LEVENSHTEIN_TOLERANCE; - if (isMatched) { - console.log("fuzzy match", { domain, fuzzyDomain: x.domain, distance }); - fuzzyResult = { result: true, type: "fuzzy", extra: x.domain }; - return true; - } - }, true) - .last(); + let fuzzyResult: CheckDomainResult - if (fuzzyResult) { - return fuzzyResult; + for (const fuzzyDomain of fuzzyDomainsDb.data) { + if (fuzzyResult) break; + const distance = levenshtein.get(fuzzyDomain, domain) + let distanceTop = distance + if (topLevelDomain !== domain) + distanceTop = levenshtein.get(fuzzyDomain, topLevelDomain) + const isMatched = distance <= DEFAULT_LEVENSHTEIN_TOLERANCE || distanceTop <= DEFAULT_LEVENSHTEIN_TOLERANCE + if (isMatched) { + console.log("fuzzy match", { domain, fuzzyDomain, distance }) + fuzzyResult = { result: true, type: "fuzzy", extra: fuzzyDomain } + } } - return { result: false, type: "unknown" }; + return fuzzyResult ?? { result: false, type: "unknown" }; } diff --git a/src/pages/libs/storage.ts b/src/pages/libs/storage.ts new file mode 100644 index 0000000..9b87fa3 --- /dev/null +++ b/src/pages/libs/storage.ts @@ -0,0 +1,73 @@ + +import Browser from "webextension-polyfill"; +const isUpdating: { + [key: string]: boolean +} = {} + +const lastUpdated: { + [key: string]: number +} = {} + +const currentData: { + [key: string]: any +} = {} + +export async function fetchData({ + key, updateFrequency, getData, +}: { + updateFrequency: number, + key: string, + getData: any, +}): Promise { + + // return current data while updating + if (isUpdating[key]) + return currentData[key]; + + const timeNow = Math.floor(Date.now() / 1000); + const lastUpdatedTime = lastUpdated[key] ?? 0; + + // return current data if it's not time to update + if (timeNow - lastUpdatedTime < updateFrequency) + return currentData[key] + + + console.log("Updating storage data for", key) + + isUpdating[key] = true; + currentData[key] = _fetchData() + return currentData[key] + + async function _fetchData(): Promise { + let data = {} + try { + const cookieKey = 'llama.fi-' + key + + let { lastUpdatedTime = 0, data } = getDataFromStorage(cookieKey); + + if (timeNow - lastUpdatedTime > updateFrequency) { + console.log("Fetching data", key) + data = await getData() + setDataToStorage(cookieKey, data) + lastUpdated[key] = timeNow + currentData[key] = data + } + } catch (error) { + console.error("Error updating storage data for", key, error); + } + + isUpdating[key] = false; + return data + } + + function getDataFromStorage(key) { + const res = Browser.storage.local.get([key]) + const item = res[key] + return item ? JSON.parse(item) : { lastUpdatedTime: 0, } + } + + function setDataToStorage(key, data) { + const value = JSON.stringify({ lastUpdatedTime: timeNow, data }) + Browser.storage.local.set({[key]: value}) + } +} diff --git a/src/pages/popup/index.tsx b/src/pages/popup/index.tsx index 9a099c0..02a701b 100644 --- a/src/pages/popup/index.tsx +++ b/src/pages/popup/index.tsx @@ -3,7 +3,6 @@ import ReactDOM from "react-dom/client"; import { ChakraProvider, ColorModeScript, extendTheme } from "@chakra-ui/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import Popup from "./Popup"; -import { protocolsDb } from "../libs/db"; const queryClient = new QueryClient(); @@ -28,6 +27,3 @@ ReactDOM.createRoot(rootElement).render( , ); - -//@ts-ignore -window.protocols = protocolsDb; diff --git a/yarn.lock b/yarn.lock index 429ead2..a325ccf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2591,16 +2591,6 @@ detect-node-es@^1.1.0: resolved "https://registry.yarnpkg.com/detect-node-es/-/detect-node-es-1.1.0.tgz#163acdf643330caa0b4cd7c21e7ee7755d6fa493" integrity sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ== -dexie-react-hooks@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/dexie-react-hooks/-/dexie-react-hooks-1.1.1.tgz#ff405cc89e5d899ddbac5e40d593f83f9a74106a" - integrity sha512-Cam5JP6PxHN564RvWEoe8cqLhosW0O4CAZ9XEVYeGHJBa6KEJlOpd9CUpV3kmU9dm2MrW97/lk7qkf1xpij7gA== - -dexie@^3.2.2: - version "3.2.2" - resolved "https://registry.yarnpkg.com/dexie/-/dexie-3.2.2.tgz#fa6f2a3c0d6ed0766f8d97a03720056f88fe0e01" - integrity sha512-q5dC3HPmir2DERlX+toCBbHQXW5MsyrFqPFcovkH9N2S/UW/H3H5AWAB6iEOExeraAu+j+zRDG+zg/D7YhH0qg== - diff@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"