From d1a430fccc8d8b21377c080650d1f9e48e6dbd80 Mon Sep 17 00:00:00 2001 From: Krzysztof Modras Date: Tue, 2 Apr 2024 12:47:26 +0200 Subject: [PATCH] feat: handle redirect rules (#15) Co-authored-by: HoJeong Go --- bun.lockb | Bin 63504 -> 63503 bytes package.json | 4 ++-- scripts/helpers/build.js | 6 +++--- scripts/serve.js | 2 +- src/converters/abp.js | 11 ++++++++-- src/converters/adguard.js | 42 ++++++++++++++++-------------------- src/converters/helpers.js | 12 ++++++++--- src/index.js | 24 +++++++++++++++------ test/unit/helpers.js | 6 ++++-- test/unit/trackerdb.spec.js | 13 ++++++++--- 10 files changed, 73 insertions(+), 47 deletions(-) diff --git a/bun.lockb b/bun.lockb index 121cafcdf5fdeedc91bfcc94a60e9ece7c944e78..da0d06cb09f375fda3b5b1202ed2b8ed75c444cc 100755 GIT binary patch delta 4988 zcmeHKc~n%_8NYX!L7pIph%k(Zf(sEDX9mbH5Pe`=lIpl5#w1l~kX>aFhz83tBt~Ns zm*i`XT4SmvQIEA69YhvU5jfGLiZxo(8a22jjcLR!k)&#W_q}=AbDH$@^tAn}1LuD4 z_ucP)`@8Snm7QKII=xn!2b})Hy)P@Bo8RrR<;>Zlqw(SIyV5G#Mt|rxeYPcdWlo2y zYUiYCZr$j|SP)~bp^UMfp!yz+bqCEZ$(o;4P+IZ|Z~$-!=7rp1;<~4e(;(^#l#yGw zprDwslC1psS)dl0rwggN(o5A6;^cyQzDOJK!pp%S*68D0m~{bj?E~{%u4pm z5A~?%3o2ak_t2~YNaYpe#73xkUIG=dU+$;M-3?Pixdkd*y$C8?JE8KcTtfz^j00dG zzpjjQ{N-jbTcjHM5>Sz}V;&j_Dsod4o1b0;M=45`y8$W!_ySbOhj`?@L50D;!j1?u zevssj*r^GASD zd)FRN^p2~_tgc&+dEx6)P^Gazkq_5I_fEq&Lqf`gBYKVyzg7Zetmt%YzQ8FprxiM^ z$*vQ4OJex-(R#){YsWmr`i4+zOuYLA-`Dg89i{{5TK6Gj@%3>h_WYd}?@rZ$F)Hk> z=h8^n>jtS%jV!$5k%9N$3o zp)x;7T^M_i!zfE*&=*3lLCi}PK*Dz&{I*MtK=BF^YyV1)Fj;coJx>A;+Jh2(0EB>- z;>Knm3s5(z_Onah13d=BsoK}hAEh#rER|q05%OG>2NJ5hs5vUp2Vk5!)zdCbLX$lL z#KdChRiM%Lm$d^4k9E`>YUka^5iUzIHVff5SEB$DX1onHPzRNqgQBGD|K*kSWIplOOmWP;aHS%fUz22beKDOQG|`9y$TRNXg`9Q|eKYj7U; zX~PEPK>>DNO>npk8c2}RTbt=dl32xl16L`YK!uSL>~aPG0-N4jX-0mA~0EE z1J(2>FOW0}NL7b48(acW6H>a*BZcC@J%;V{ekl#3fMlwONJbOX$H>w#V4(xD1MHIj zV69Zu5q6$LWy55t3_Ot)yxs%s1~<@Xy5pbBgUAspOY^Xe3hio^-UAYTtMA}NAW=*; zF@v%FiU5^{=u41ZA~&(Ix_kHe-ZJ87hGiMGnyZ^}}W9OW+5o zj2^0b0ZWKt29VIB)X}g7$U-9qC5zUv$lQ+{R$01)jd&(x3dL*5M^U|1#)i~|@e(;~ zvWOjHCDq$x!zFC$lc>p-Y>3BRHeNhQui?p|+6YPX0Z3MWFwcfF=F|{(k|SQ`yWM5+ zy?s*>_ontAxo}%+$)1aIY`q@$o0s3ehiTc)w2KLrr-sLWJJZsB*5@9_^VbY>SkQ=3pd)qO;M%sb`N?opEh?q04PcK%Fa$goj&Cgr$nepI*r z`@x$}f3tki@Hd_`{qg2gt&wQ~_0vP|u8evkH1^4(6Ka}|r>-e(+tvE4`>C-hhLoD@ ziEI8OM^8R>=}>0$=!R7%e-?E(EHwK1d2>YX)U!#qYi}5L?>w=#Gws4NznX0xGSs&@ z#=Q=e)RKO45mUmxQf} z+q&!g^=-35SlZ+nuPn&1zV6uc`KlX!d6z%_>czHg#V_tUv5uBGLsOn#+R-n3$l#YE z-X1&mi@Z&Xo*lw6=L~loZCvx|&4YJ)zBP8=-h~m3Y4umWoVI@Zh?WoU-8t*uaA8!) zhM6-eip#tgxGy=k4GsV4FO+ZT?Y7!{JyU<(LB#j-DsyjtA<2vDDX=^!SUaj_%*`{; z_8c;lnGxEdRh!XaZ6AO2*rD-+<{#uydbufUBtC!mcZro-@c<88HK08~#maw|C-mjd z$;>M*Vr&(4ln3!28G~2Bl&~sRXvgcT6u2meUv=kKEa5zxtW`n(;#gf};$5^8^8X+I zE(29v_;&hwlKd^UL>#DGUfAxmGXIx(X)Y9({U?q5K)0wp^S>G0%U-?~-QQ-mnXL&p zYa+NvWOVTI%8%w$j;+&it2x1JVeYrLKdIAOEoQ6x*83xLChZRDxAoYuB>h{tI-Z8P zcu4BW={E9LD90VgGifcxQfhZQ`CF@$MNID8;txjGGUxg2@P?47v*6* zKz`{GR61@^h5@4eTqD_xE55pN5A>b!RG+&dwyob8%+*b^UW=1srd z7l-BqCA zZdbWWFYg}rKvpR%)isCl9QwFS6`Kk3C&B%|*{NA{t@ z7Q51wee|f(gN8Knt#qr!$vf%MRwtiGueTcUy}8pC5}v@coA9$)tIZ!Ju1Hn#Y86#p zPv=^V3EGW%Sm%-;|AX$`%Di1Pt9BP(n|c2Dih#{6m_hc;He13WifuER7UM(R3q6rG z=0MCR?#wHgPZ0fMWmS~kX5=gB=aABF?%PIAv3$4g-ZCA>`B3@op&f0;1nnjtTUl3f zsLGn4*mnr~+MRx5!UW6KRZU}*nRqdi?Y`4Cp7RCn$3HmCyJ4p?m*y;_dq;xZ=EG?Q hz1Y;;E01>5t6SIF<3-#H>nPwvI=8y#o%nUs-vFu$Q6m5V delta 4935 zcmeHKe^gXe9)A~RgcrX=L>NXCMN~v4hJhX<>Z3%em62R4`vn`q2&3{tKoLV{a8uLG z)V@@tQYjrPbu}FkMnyr?=hM>1!aZyvEKOv zuZn0;?n;Q4+DjqjC@nG#R`om!%47d>kSh0Mlp4xSQ0{6sD0l56mG5+o9HKHhz(9Uo z1^M__Y!@s;)!3f{X`Q>o5K30|c29yVI1(eH2dgc8c zsgAi}FIXrLTt~p*3rcuq@LPrCal90idu#^f5l4e^2em3|)jc~xHLMk_7?>N{GE$}2 ziRlN2=7P($R?eC2s6tcBC9^)lVTWQKOjm*+M1$@F6#sWn?TsiI?`up|tT+m2*Iv|qL0s<*BGQ~YV;CUTo4oq>U z^*!#ZN3g;JR2MXYu7!(kYkyxBBx9`~*svjlC{yJD{z1D@;2g|=DFlO?K;W_v@`TGQ zTNxY3IzZN+LMZoBTX;H)ArJ7YaBVWR^-X7Ak~KmWCnKY2(Bn^amNb8WWB?yx{Uq_* z+o&!`V)0~+l=ah*se7m`ek^$+W%e=kU<@X!K^C)6V6I3HXC!eE(0D~tU8p4P14>a5 zH1(1-N*2dp!;S&3k0M_M#3g(c%WnfE0r^o~pd|hP^Z*c}x&Vn~QjJj-t8grEd8Wz( zaa~w#kl1yy4wS`*&?jTT6Nts6`2%F$7C!~VUG<@kNQwEAHCh%ku-&=CFv8sj;_CY- z>brq56nQ=uhe9L+>6F#=R5M6s-Q*c0i?JxTH|Y2@e}G3-g`pB_p_;+6*aIF{>!TP+ zMECHxe3iIXk~Kya{|w$8M=dyGyeLFsi>U_^?cniPRY&#MK|E7x)>DDFeq;rCa{&Dc zNF-i^LL3%c5`_+s#Cjk;hX992>@s=Wu4CbMv|#>wJua58xV3zo#D+vr}T z>IU`1$v$a=>DI9E`qenfGN?B)T|9*+bxB8XW0Z7${TNt$8ap(uBUzikW32) zr;Cg6lnLKZDMWZ0i0i_dA(9v}{C3?2ODvsgM$6&?@OXZ(uZKwbT|k-C8=TJilQmHm zD{w4wD{8(r0dfD;o!AY;%c^E*1ddZ4snWl!jA~4>*bE+T1GNKv;{_pGSB(YUyD-b6_<3nHso7BAso z&4o-E-=u5;dCW4~NIe)Yku_Q7v14?SCt22C!XceQZOQ5SR2;I2PgcbS+D<$I zl&&CdpWQ78HVv`KWKEITZg)*e|A375X8tXA#FaB|#RPv*-M;0A$EtT%Uw(D|j2)l% zn;UZU?Wl24x~}P=*`<#+4XjyQYse0qnpZcczG&*X;@8%^cC2N|X7>sCfj$}52g~cf zFVQ`JW#v-|&mGs9KWo1J;IaLq1COjL^=x-&5l!&e4DMqWh7o41LDM+L^gye|xL_^7{AexknwJ9-leq=If8WADXrI#`(Cs zn`=8?Ub53H%?mmHU1pxA_|4+EC$Bk5Bc5s5(DuopZGVmMDFClolh3lS`$5O^g`}d zYs?nAErnLvA~SaGOTGF*if!Vu))^lEDNENzoEUQX)AJLb*WbKa|JQuy##!%(lca`$ z+rE3_;}xrp#66MQHuA}dr$?Q>b-Ht!+#rdZ<{~H zm+2|AHY{BG=q#A+FwOLSE+{kOw9ij%#s_O7@VXOGM~%ld|16L4YmIgpZ#?|D_{uGK zfCsK-(7vF2<-g10`buUMIx5Nqp@A;chOu85!y90VTjeXX59MnV`fM1x<}O*djIkmz zJH!6PvCe5^w`d3C|3CgD1I}A`g$8(&{2dNMd`h`=IOw!8|Cf1rE)-YzPa64|Zc%&Y ze>1vQF1bE3CNJ4!Hkr-BWCS-986Ez@qGPibWwrXSJhLg;l;nPG+f*N&)g&dmZ@fFs z$Ee*V12>;KHAc6-7;;G_bBZZR(38z=V9Tk$2+Bnt{)tr zvj~DBVDw%nU2pAS1&w{SZig}iZ88gjlgu6)TR~1wf_Msx@ipn1OC6mCwunx6EbI)i zJqC7xj2Npab&rL)$$_zh8uu8C-b#z1?&gD{j))7dln z>s^YK&Nb~j+2oB0Z*=Xp_|l%X?A-&!$;ynygqdh6*=sOrx63(oSHHR5xUfK(Pd4Gp zU@UFeYhYQl7vog=Xs?Z(qu6~#EQ)@&?Ue5UKK0=d~;3 zzU;hOVdtY=-uO^6Kl=TC3+jIXTC|Jj^!`H|?C%cRM zbss2V7P7S)jM^3W;Y|y+4XPdaCN%Q$!tQCRYd5fZS_diZ;_JVBg5UY3jpOb}eM2YP zZ7h|dJ1p!7P3f>0wcBWMPpwNQH{X9pRw*pCc0{paI@7TZeOPwzP9KK(TU5jU*p6H% zpwsYAdhsmD=sVt#1E{Q1Qo3@04pq9-kOsDeZgkpkjhb-C#&T%QAp_nwy@w;BQw8l# z`?$TqbS!Pvy-HrOy9Hqboj+tq)$YcluPqA;KIm>%<|W>&+O2wB;l&SDg}l{?8D!6t zoSfQ4iH8lwWq48dLr>&QY)?4rF8mtvsl0#8Xp{WI2DXNtfs}R!-#UIu(yhiHSNbq~ z87jY4+J4xOs@=&G7qwO%b(&KZ`&MpWyPa=L%}(0 { console.log(`Detected ${event} in ${filename}`); try { - await build(); + await build({ debug: true }); } catch (e) { // no need to do anything as build logs errors already } diff --git a/src/converters/abp.js b/src/converters/abp.js index 0f34a62..141489d 100644 --- a/src/converters/abp.js +++ b/src/converters/abp.js @@ -1,6 +1,12 @@ import { FilterParsingError, normalize } from "@eyeo/webext-ad-filtering-solution/adblockpluscore/lib/filters/index.js"; import { createConverter } from "@eyeo/webext-ad-filtering-solution/adblockpluscore/lib/dnr/index.js"; -import { normalizeFilter, normalizeRule } from "./helpers"; +import { normalizeFilter, normalizeRule, DEFAULT_PARAM_MAPPING } from "./helpers"; + +const PARAM_MAPPING = { + ...DEFAULT_PARAM_MAPPING, + 'redirect': 'rewrite', + 'redirect-rule': 'rewrite', +}; export default async function convert(filters) { const converter = createConverter({ isRegexSupported: () => true }); @@ -9,7 +15,8 @@ export default async function convert(filters) { let nextId = 1; for (const filter of filters) { try { - const normalizedFilter = normalizeFilter(normalize(filter)); + const normalizedFilter = normalizeFilter(normalize(filter), { mapping: PARAM_MAPPING }); + const dnrRules = converter(normalizedFilter); if (dnrRules instanceof FilterParsingError) { throw dnrRules; diff --git a/src/converters/adguard.js b/src/converters/adguard.js index 0cde4ce..c2cb6be 100644 --- a/src/converters/adguard.js +++ b/src/converters/adguard.js @@ -1,32 +1,26 @@ -import * as AdGuardConverter from "@adguard/tsurlfilter/es/declarative-converter"; +import { DeclarativeFilterConverter, Filter } from "@adguard/tsurlfilter/es/declarative-converter"; import { normalizeFilter, normalizeRule } from "./helpers.js"; -const converter = new AdGuardConverter.DeclarativeFilterConverter(); +const converter = new DeclarativeFilterConverter(); -class Filter { - constructor(rules) { - this.content = rules.map((r => normalizeFilter(r))); - } +const createFilter = ( + rules, + filterId = 0, +) => { + return new Filter( + filterId, + { getContent: async () => rules }, + ); +}; - getId() { - return 1; - } +export default async function convert(rules, { resourcesPath } = {}) { + const filter = createFilter(rules.map(normalizeFilter)); + const conversionResult = await converter.convertStaticRuleSet(filter, { resourcesPath }); + const declarativeRules = await conversionResult.ruleSet.getDeclarativeRules(); - async getContent() { - return this.content; - } - - async getRuleByIndex(index) { - return this.content[index]; - } -} - -export default async function convert(rules) { - const filter = new Filter(rules); - const result = await converter.convert([filter]); - const conversion = await result.ruleSets[0].serialize(); return { - rules: conversion.declarativeRules.map(normalizeRule), - errors: result.errors, + rules: declarativeRules.map(normalizeRule), + errors: conversionResult.errors, + limitations: conversionResult.limitations, }; } diff --git a/src/converters/helpers.js b/src/converters/helpers.js index f2d51d4..85f0b4e 100644 --- a/src/converters/helpers.js +++ b/src/converters/helpers.js @@ -1,10 +1,16 @@ -export function normalizeFilter(filter) { +export const DEFAULT_PARAM_MAPPING = { + '3p': 'third-party', +}; + +export function normalizeFilter(filter, { mapping = DEFAULT_PARAM_MAPPING } = {}) { let [front, ...back] = filter.split("$"); let params = back.join(',').split(','); params.forEach((param, index) => { - if (param === '3p') { - params[index] = 'third-party'; + const [key, value] = param.split('='); + const alias = mapping[key]; + if (alias) { + params[index] = value ? `${alias}=${value}` : alias; } }); // remove duplicates diff --git a/src/index.js b/src/index.js index 973b99c..1f7f1aa 100644 --- a/src/index.js +++ b/src/index.js @@ -8,12 +8,16 @@ const $outputAbp = document.querySelector("#output-abp"); const $errorsAdguard = document.querySelector("#errors-adguard"); const $errorsAbp = document.querySelector("#errors-abp"); +const ADGUARD_CONVERTER_OPTIONS = { + resourcesPath: "/web_accessible_resources", +}; + $submitButton.addEventListener("click", async (ev) => { ev.preventDefault(); - const rules = $input.value.split("\n"); + const rules = $input.value.split("\n").filter(Boolean); const { rules: convertedRulesAdguard, errors: errorsAdguard } = - await convertWithAdguard(rules); + await convertWithAdguard(rules, ADGUARD_CONVERTER_OPTIONS); const { rules: convertedRulesAbp, errors: errorsAbp } = await convertWithAbp( rules ); @@ -35,7 +39,10 @@ window.addEventListener("message", async (event) => { try { if (converter === "adguard") { - ({ rules, errors } = await convertWithAdguard(filters)); + ({ rules, errors } = await convertWithAdguard( + filters, + ADGUARD_CONVERTER_OPTIONS + )); } else if (converter == "abp") { ({ rules, errors } = await convertWithAbp(filters)); } @@ -43,8 +50,11 @@ window.addEventListener("message", async (event) => { errors.push(e); } - event.source.postMessage({ - rules, - errors, - }, event.origin); + event.source.postMessage( + { + rules, + errors, + }, + event.origin + ); }); diff --git a/test/unit/helpers.js b/test/unit/helpers.js index 5900467..55869a8 100644 --- a/test/unit/helpers.js +++ b/test/unit/helpers.js @@ -13,9 +13,11 @@ function normalize(rule) { } export async function testRule(rule) { - const { rules: adguardRules } = await convertWithAdguard([rule]); - const { rules: abpRules } = await convertWithAbp([rule]); try { + const { rules: adguardRules } = await convertWithAdguard([rule]); + const { rules: abpRules } = await convertWithAbp([rule]); + + expect(adguardRules[0]).not.toBe(undefined); expect(normalize(adguardRules[0])).toEqual(normalize(abpRules[0])); } catch (e) { e.message += ` diff --git a/test/unit/trackerdb.spec.js b/test/unit/trackerdb.spec.js index 0ec5aa5..4a0deb5 100644 --- a/test/unit/trackerdb.spec.js +++ b/test/unit/trackerdb.spec.js @@ -3,6 +3,7 @@ import { test } from "bun:test"; import { readFileSync } from "node:fs"; import path from "node:path"; import loadTrackerDB from "@ghostery/trackerdb"; +import { detectFilterType } from "@cliqz/adblocker"; import { ROOT_PATH } from "../../scripts/helpers/paths.js"; import { testRule } from "./helpers.js"; @@ -20,14 +21,20 @@ const engine = readFileSync( const trackerDB = await loadTrackerDB(engine); const UNSUPPORTED_FILTERS = [ - '/baynote(-observer)?([0-9]+)\\.js/', - '/facebook\\.com\\/(v2\\.0\\/)?(plugins|widgets)\\/.*\\.php/' + "/baynote(-observer)?([0-9]+)\\.js/", + "/facebook\\.com\\/(v2\\.0\\/)?(plugins|widgets)\\/.*\\.php/", ]; test("TrackerDB filters", async () => { for (const pattern of trackerDB.engine.metadata.getPatterns()) { for (const filter of pattern.filters) { - if (UNSUPPORTED_FILTERS.includes(filter)) { + if ( + UNSUPPORTED_FILTERS.includes(filter) || + // not supported - https://gitlab.com/eyeo/adblockplus/abc/webext-ad-filtering-solution/-/issues/572 + filter.includes(".*") || + // ignore cosmetic filters + detectFilterType(filter) === 2 + ) { continue; } await testRule(filter);