From 9cf778c1e71d4735c2b233942e7cc03ada5b3724 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Sun, 8 Oct 2023 11:35:44 +0200 Subject: [PATCH] Allow selection of JSON files & updated error message for wrong files --- public/port-0.0.0-py3-none-any.whl | Bin 8464 -> 8592 bytes .../py/dist/port-0.0.0-py3-none-any.whl | Bin 8464 -> 8592 bytes src/framework/processing/py/poetry.lock | 265 +++++++++--------- src/framework/processing/py/port/script.py | 32 ++- src/framework/processing/py/pyproject.toml | 4 +- .../processing/py/tests/tiktok_test.py | 177 +++++++++--- 6 files changed, 291 insertions(+), 187 deletions(-) diff --git a/public/port-0.0.0-py3-none-any.whl b/public/port-0.0.0-py3-none-any.whl index 7c9552de933a8f7baab26dd01ce1215fefe7cbbb..e13fbe2761b4cc236f6bff29118e6a5728c5cda0 100644 GIT binary patch delta 5768 zcmV;37I*28LXbnSK?n+n2WlHw6952JN|Q?nDt|aHFdHW~KFnl-v9rL9AP}_Jk~pGB zf**D?fq{LHyO-N1xvJ{#CMin^ZiK`Z`=h$Lx~lqD)lKL7x@tO>cf9GUs%U4qD8L_a zTIX@yRCUYJmeH5lx~cZ2a#ij3X_>W5_Ie|pZx3z9_t#aKc05vrA8&r)WyYJC9s&we zK^HnxWxwb3Ixo0-{H?7@^*sY(UB2gng_8^k8h_os?D(gy$nRD2wy*1kx9x29V)8S4 z!S3?iUA1G#6^dX5J@RtHUL5N+gVrF>r0r4|=t{F5t1sbqufMy982+29#vq)jrhipU zXR0(=SIs^}+QG~Je)V$y>Se}W{&4y7r^}bO!3^5*HORMVd7|k&q(#r?moo;RhIf5a z4u1!W+peii>xg;d%_Zp#?~=R>p*Br~y#Ku8honxstsDW`vmMJzrpiSzesnJ}UGY5z zlHo35@2ir}b<=!pwa#0rHH560V1+j`(aU1E$lLB1%b*OPKETwm+07(dvhjc?IsRg4 zb0ZpCtRU4tij~h#M_GXt2YvpdmGU<5hCsRFCKiNUzY$QewunFh}nq7JM&h1n~VINQV7g zl3_nUv#K#QKf-fh8yq@YUUtupC#3>(-iRK7`P$mw2P2@hC&C@^EP*{?--h(lGJk4( zM!TkRBS3T#;O_P{ERE-Qu;n7ShR0y(n6~NjAI$NXFpR1D8#4~lwSDyIdD(Mac*uE? zu~7Goh)Y+7JxK8WAf^fmV;Z*+`KonauHD=5D5tGsG*#%upAJYkQ91o zv#W+VPE{#_Ss~B{BSo`BhRi}u-bj1CWPiD07q;#SjzZHd78*FW7ceaf(0|YdEXr0) zNg>P^ua`ngwTi0F+p2Gh11&j1MhNJ1e5>{p^ zdr7q}sx(WweAiVwu_+;D$4x{T1Qx=eSR7+xtASelvg*h@`WDo%}y zHo+WSER-0wP#ut6secJEekZ%|R!~k6t7DLBi(`dBj7$YyQ~V6}HJiVBx$I!j)_R z|CQ_>wlRii2-Xx?6srqTPI2lrEVnKd9;rDP&S}0R9Wf4yXMa5!Eet2LmJ{KhR=n;Q zO)$<#K#eb~hR*^g~Ud*POv65Q$sD(d2J;K%E|mp)Jhy&$3laF@gnABvic@ zuz=)9urgvJ;kRfU7U}*zOKF`i=_kUaCGLPS2zPJ{XlT)A{JA02o-IHH*pFIV0WKc5 z@Lj4_hoXYz0Z@wQ_1|QagxEn2mbp0KB=)}n|5ife}jAKzNq4{eBPFcMu(E^_7Ou4F!8iYM5n}DI_&zu6iuF3AY#lDr}CHtNjjV=N3a)v( zYVw+V)+Jjw#DaHKQB;q-0Z%hl0sEI<@neqBo|2beOYwfo>Zn=!-~UFsy0mW$=jE zIi{g1A?ZP>+0Z7d+zoEB8EtsO+5*@2m+?h7NfxwFI&IW?38ZtSiRz{Wc>}iiE`6vP zFv`=(os@QR2^qKMmb)J2+_{{TVH+s+)6CwC-ioaSmi8CQRvNttB!HAo#9BH!Dv9qJ zP=9G~-!c*)N-gKHu^20;!N%4Ev-NlW zNJA#aMzdp*6UK79%bqtIh`ZMp5sn{0%((87s=QLfn|!l%3bxhbRZ#V??0*3aRW)j? zukkBqx-MuIS~hy3w{h2GAT~|#n17>sQ2L_8vkYhL7XG43l;M@e5c3>rZVvCTjl+rT z=adb4o$MNiuIe;Dse@TOo|65PyVP~yBSh6HPE(##BnYoslU7+8A84QQ9h>#=*3>3uPe>MUR0(GkMT9Ct9Ih zE$DSsmllaQ?QO$3`v`OOH_XECT-bru8cL?qPkDQFF;=h1n}0@UhM1NnpJq+B zqQ^vNiIv>c@srHSOSRb+<)%#yct1NakHl!(;SCV8RFFb^I*n8!PD>O>HU-Q}U^|qg zBw~5N-j;3OaCU(~0-k{1T&u(N3@DbMjBW852x_X-O z1(^|7ahUCEg$nUnAnbFmSdjx?8!E?QROgn( z;tN;IQ8zX0^2}xQ1#;FtKC))ACQTb>(WchwbMS%E+Kf{0Mt>|%$Fo|951gddS|a_# ziT=$JI=N)omYMBe6^&#ae9A|cRXYpBS()+@Kj!6b%1gM^2Yx}&`Fh(PkOhu3RY69S zo$Chd1vw*DZkg<%^yWV~7Qmtig3}sv%)0HePI91acyfG_xk8*u73o|oAHB@5w(rLbuRplwL9{SV3EdzhlK}Fe zUIAgZRIAIdg;^r^k`vw{6;xV^vGqN0F^{Rc{j(PgwMOpx74FE@(x;E{Z%V*5 zgxu+%o^Ar-p;K0IF=ugf&p*|m2{fBF#*=zA#_cYz@2dA8!P}14gAwW!UxAQqQ7G3^ zCLhGh4pk$sae*x*poMvtBmm)yb;N2gUqq02J#x!pfx43cM-f?(fRb;Z~~ zesvodihmf=>j$@yQIRV%tlPw}0By{J$4ygh@RAfXdF`>m@p`#99bU-&lisZm!XZJ- zy(>ui%9Sg5?~c{u78el%{o&m!x+J8p-HThz^cIWac3*eG0HsT|Kh&nEHsLyuGC^2N zN0{?LW9I^u{oz`r4I{S9<+UlpsJIF<96{cwl7H9F@J`mqggw5+4DlWugs38TVO@o; z1!4?RerPP-{u!AQekNo_6vFW?)?Pivzh|k>&LI~aq^-j4wE`#;F~JC}2uOqYnV15& zw($5gCT{KFXY>l8o>I|HtidrkOm@oGBib_hvX1?M86@c@&C6*GKek$OU`=IuAp9H^ z$A9tJsn3L1@H1j@ps;PfLKm^_i->J1)L-BN&$;O$_j!6Ndxir!&mEU!gy#7rD)c={ zL$Bi{Y7b0mKoD*;=D6xeQn9TvbXv?K*PWVZ%tr?{KXF2?JxS=joDb;%FK41(34}j0TeAy7QldEl2L0i2Q z)5d(LdWI1^1uXak@QFXeVkI=?7r%{3d9TQj5M-} zcJ*!gfR|xEI}(1P@nSZ1kA$%DI`~IbY+&FK?m4ATcldkDVOR{OU^3vyS|lXxG;u=0|+oDLkJ(JX#k{O z5(qMW-){iYt^-dIF4<>E4@=uoPsuaIv9cke(RcQhn*jdR%g#eX;APCb~Q zz+F)%I-*w@RE@a|Kr2IihY5Cj;p{X!y(aTA*?nY9&9;1(_0gRUUoMz7eVQHD$N|^W zGkzfq(JoXbDFdCG?CF(fqklo4K~?O80Y@K8&!I2wD_Dy9Vzqp|K$ zzwfacd2HEZj;Oi??KB&)iX&0}UA{|Hib|`s$Ee^h5UX<9}-|#k(GWLBx9q zV!IC5)ACRAnJ1u8QP_N09i$(CpTY&G*$m26zGE}=a8ULZ_BQ9F+xxQ>A$Ig=yS~RL z%6*j`A}uv_M#j^OPPLq&f1uKEdap86#V(~SS?#dvs_KSx-%)9(c{n4_3<3omBM4!jmVZ72*IlSCH|~#kWZCtocLnhE$nmu?gv(NQCGJMN!X%>Kr#sH* z*w4Up${wh<$>_ytf|&A;YCLWM0beUlYY=2 zktywad^hE`>VGqQIZ>@npoB}m{vYtD!Hhoc^U)sijA?45xB}^*BS6QS|4v2pTOz=7 zfM0-Fe<_L}vgH{uAJ9&N)N}CZmt)h@ap?*Ar{Up3AVLoxs=o>#c13z4kIqcQUs!Qp zaU1zVNy8pMFh^X2YbZfulI+10lSCp$^{0#ut-K^8UQeWl=QwRP4|Dc6`z`vq$xRN) zIS^AF78gr-Am}E42AFh6*{ian5MazWwb24Z6j>ZyGT`6EVet@Y01Bz zGRB53LZ9^X-YXS6VZrK1k~u{OTU(nq4CLUNq?o_?#%h_5DFl?+L9Y<5IDDcfxO2Hc zr!vr`h(-5dOr8g`pUNePgF|2%;f4`TByc2(zo?>sJUSq&P)gZN(T-+8J0$Z?UlR-E zs-JaNJ+y6O>y6!-=Mi(}Gm~oflDuFD96h{X^wZ+V>HOzt>6(9~j&spu`U^8WJhhtE z*6YV+3AZxelRX@p1-3x@ou-rB94CL!OOv8700iLs{1%u2Bt#E$KmsDZzz2vI3KUEb zLLi_FzkX}V+FcJ;-@oc^RyFNc6cI|=DEeeS0>`lev~QKT^big{sFK@Z-WC?H2T4J9 z1R?G6Qv)KOa+D9nTou&G2m57WM&42onI6l>_H-AE(SBYAB5;hf^V03D1nYklbAr=s zgm;~F5TyUoxhl(;Uigc2A38(JQnuEi-A7xupM-YrcFOT;mLfQ6b)wnVxgW0605+wTR{{V~eM)6a=nE1)hU^qa zDn>zb%L8|Yu{Y6o=Y^%UPEdch3x`_;Xc2(ApsJ&%!Osn0?Tb<`FInLvT?oQqg$Z{v zN1jf1<51LOmIb5@DfRm9Jd(K}PbB5JN|Zi-b0NGH3C*gnmATiEzSsX<)zjR)TQ%{u zskbux1x&;%E>iQ@&SuT3vfQ&39+yVBt^@^2@6o_E&E;S5qGb1cgCHnBa(^(eQD@Tr zM{*K-F^!ku10^#LJ-j`2fFz8^@4>BUY_>=cKK=l+ryffI3Wx`48(0$n098tp{2wwL z97VNZ$p8QV4gmlF9RL6T0000000000q=B~^lRg_ClW8C-0=pcOJ{uR4a2y3JP)h*< z6ay3h000O8001FKwm|!xrU3u|cLD$a8vp7yqL)Zv1b>zfOX7$k z34Yjd3Zss)}}&iw68J zPU}3bo2qVE+A{huTQ}9-JX}@#eOhKMlfB-E@7E7)$M=_2nRYx zUa*^dcT?>cN`)#|L65xLuos7V&43ydnzUUC16^p*WA!2Y?)7(P5yO9TMGVTBBK@Nx zovGGjT{Zg@c?U25`_;?+tCtyj`NR3kpUz)i2Q#4KYtV1g@WF#c%_Zv%?~=R>;c1!(egA&PACfxlwsHiZXFHacOtp(%m{ot;c1hXqZ+UY_8Zyiyg%Pw;76uk6JA>C? zOE3`cwJjYHTbfEC+}!2eLvojAyh?7H>K;P@*%i83O2W7v=Ga}tg6}1d06za6*|5Jz zHtYu=s~Yp(m)*1DNv!~%H=;*iKDPGf!3gN>i3mqLOJGmfw;}y+IW>QN zN57_aqd*K25bpLhER7dl7= zW1;RFk(90ud(hzRgP1B9j3I6#idE~rT*>JMLGY2(3Y!$85LAKCTaJ3zvk<5wMhsyD zq`!ej(1-??agcbQmpo&)57bBTj@W)Uk0aUT^@!k(Bm4>(|KYFWtPog((V|%*!)Bo-Z?rvMvOip~Guw8BM4{;x3k#gv3z!xKSZIF(9%UP* z-u?#=$^Ho!+uZm@s z=7%}rf1xdXi|7F$i)D3Vwnl4&wubnnb*+v(Bx2wUusionipy=;pKu_w3d9p>!+iq- zGsmaMP<2OhNNEW(UUVt(62yP5T^OOkE-0$WC9`IuV{gGryB&k>>#1#WocwG^Zjr!B zN6h=GVT_(j_VLO@@fW|xU`@TUF1H9@)5Ub1C`Eq`4 zhdb=@in>Eg8xd95U3>Azxf|w`;DESm5G-^|0404TW-_0<-Q?_GSlxdv=A~LrWUSm& z_L62@RB4uU`L3&WVpBrQj+=;b2yBEwu{lO!i$E=YS#{(deG6e8k-NRK(a>q$a(4Ce z+oq|SF!-fJ@Gb%v0rT#m=E26|<$g&%3S4I^uuk^*eaiG_Y4T!5kwi zv>0j73Q$UE05N_hckq7}PR5@PI9I@`#orHx{!4^4$L4$Mb(Rf=o~yufPyDXo9;8*+XRj9uuqO}k_( z0N%uiIwb%emwAI5{!g%gu(J!y1{`dNh~L#$HPX6ByN;LcX!6}1P^bGb(864QFR5a} zyw;X0q1Ub0=17kOlOi^fP730%NcXo{O6z=1|Ks+*#E34FN@ri@pH@Ym@#lt6dyv>9 zVY^a`D>lpfEqs5Lrq!XUV0i#!AsYJ+iTZIbl!ImNzIK%O-;9gDPwS+s5{mmp1v?^d za_+$r(G~-#Vvb)e-T|Wve0BNEfoAEsbQ8?rjWsuyXt?jC>$pzEZ>F6mHyp-szP zK>SBd6{)d`75y3=GcGrTG}Vr~1P{L!%p;lguj>GD@nkAuVNs9eoii zOzt!P-1G)Q|(t#dmYL_~p+-L6}f|8a`4LU41 z37LN)t03XzK{gLR0Tv6Ht+B>ttH|zyC8?s{m+ghDxfw^2)wgWD2yP@^!5wpeVUR$S z>*B17QBKoQv}0Y!A!@s3O~pQa91+Zou-Q6pcwT1wQ#ce(i!u=OXp42$|FGREQx&d+ zJz8U;a*NpT&pA*%Sr(bC4_ID;+f_TTJF+k%LhC+^wX6GbFjO9d^J#T+Dkaw@oA{;-0oN?VHRe7P9H~D7kG;FK;i=gUZ z*#j7=YSdU?6IV_QT|gFEK6+%dao1#^HqG!*e52=J^hJrad1vhw{-R5i;kCw)^E}kt z97Z9HBZ=(iv<-S49U6zB>bN+mlUY1&k^NM-)XyMBh)2f+O<8$Q5MJBiQni0$^^}e` zG^g z=YumM=eUkWnmcZ9B`A{04lLJrZh{(kTW0unDUaH03@&JtwaRbfU|QVmv5<%2$H1VO zJUE&Yuh4!h_;pp67Ku2$Y{P#!`v`NjG|a;9T-br$8cLzlPkDQBHuhYzH$*3< zn3g3U=S|n5$3$w054m~9k8&rk)Mk5>TQ)V|{p7+tGNWyWH^9s?K}zxIbWN!^Em0xa z6fiGA>`=0jh~)))Tef|}*%>AYc%plCsSb-XynKhgi?v3F|6-(IEc|~-+%C9)bpCsw zeb`rh3mmZ+wv158;NST9Nv8zrF31rF51RkNWxiC0k6Ju&T|^X1txZLCCMk5MLpMzh zxY4QR>RHMcR7PCGVRoz)Cj1xPBG8H9$%*)+WlyJEZ;)3Aw^}an| z3leFrf{kc9H4NAbazd`$GTB4f&0je-z@i9-(-w2cyX|v%?@oV%B&iw~42~TtMuVqT zMQiwQh=nH@sAAmkvAy*VpL&sYQUdM6ljD=l72;H;NatdCdH70*SW~PjHN0-@Dz9ZN z?)eZcl+!{tzqmwZ0WW{dur7%a0cgw@PnxFM;9VUc zdF_e8@p7{`%U#Ibj&Ak~<&Yu8BNH68pL%HwtB3wO>S2RGV-eNL3*=rE|#nfVgWO#xEv`30i^-`{IjhYc=xP zD<-_&Vk3X9y$qRgvQ~BP4DVfy=*7}J5Ptml^T&}Xc{&itEugI`Wx7t<}(9&&d zx>tXS$^ApsGfZU(x#Ks$5&SM+@tD0S^3{(0`(OWqQT5;d`d2YQ6}aIea`??THCB7b)_499k-ll{Pw>UoRL z*Y=>s*uOExeoeov3eJaPilEYh)nNR<+76f)uf56XDK)Z0yyqN|g|G}{Yj*PH1AJj_?l-9C*WdX=poGciZt02Oa( zRIfv1$vNU3qhN>b?HL-`^0jVz!pez7gEV1bw6Yvr-Lm{XvZiKxzAO6ZPKS5xr(K_B z$2D@m_4J%yC_~VN$|YqEag{y2@_aOie4!+$9m}r1G^zS66Ddo?6`t-W5l4U1x>}`+ zz+yCwJQ(*qQ6o>RXe@Ozi)BGOEk>+TG}M2W?-Et3(Pr%lD&z~qug%euAu0p7pl9EH z&T9xPAkViTbB+feBUPZ8st}p%&{NbP(HxqX>hbgn`S0lFFZC?9OC-D z*qj6QwEW$C=BZqa#?#D1t7>-`J!usg=;V!#;0Zaw zYOQ#2n44NG>y{WzN%4>tIKE?}$%SZM5l9`$@vI1E14hJP0uX6zr0f#~I)odInM$Gm z35cAs68eK7rG3Zks@zt6hCjwrIuDd`>Fxgkq8b?KLqQ(^qRf9-u|t$AqW1sd_AY4{$e&E{^-{%rqZ zT2v&*4vSQW#o0Jg9&5NYGFi*4McD+jg9j=QDL*UEz&|w5Z2G+b^Zy4>O9KQH00008 z0000X0KgdY$>0D001E<>&l@9um61V912GVV@A(yTROp7?R;Y%ySlr&KEDF63+c6#7 zBsIxa_TM`d(R0Wp%=g~#^72d>kC?GKP)r=Wfw^wf3pz4z4l%30d9CHBx6wI_lYvek zoU#8vOYp_{1g)%rHh9dUhap(*)ppFM5Dn}c(*PIraHI$aBL9mja>#>W0n!46T?9SS z6jAp{xH=A{>1sgz$aYEt0l8+{d8vP)Xt`j2ZV*kz`KGgXK>U-En$oyB@h9m%bf%n*Y6U?t z&SmxpNBMMR-bfTU^r{LexmUOpavpOai%(aqso67>vZby#58|&MTIf91opd-`R6qy$ zc9fm-bRRC$KxwF4)&L;|{zx<~swX7uhb)Q-%yH0M%3wXob5GMZ$;_5pH@LNbGnd;} z)MBLQ0K)N)!IP$7?UPcU1wRt~qF?kyx}Mw;YNqsHUvj(c($OO^m}CnRSh`vdMk>^l zZ$}W{zd9AX6^bgWKNi+j$NEuonZm6_u zM1Z96Ycy~yYyK;qr0ia$j?!9g0A`z(SYtXmM zm5%{<`wz1xA4>rW?v^b^!x8`hK}eH)ATk`l81u>C0000B0ssIV00000000000001_ zfg&4|T_G2f;UF>sJsgu=As3S@Ar%~7x!IYd0RRAZ0ssIT00000000000001_0m>Yc NT_G(7ryc+R001Ojvl{>a diff --git a/src/framework/processing/py/dist/port-0.0.0-py3-none-any.whl b/src/framework/processing/py/dist/port-0.0.0-py3-none-any.whl index 7c9552de933a8f7baab26dd01ce1215fefe7cbbb..e13fbe2761b4cc236f6bff29118e6a5728c5cda0 100644 GIT binary patch delta 5768 zcmV;37I*28LXbnSK?n+n2WlHw6952JN|Q?nDt|aHFdHW~KFnl-v9rL9AP}_Jk~pGB zf**D?fq{LHyO-N1xvJ{#CMin^ZiK`Z`=h$Lx~lqD)lKL7x@tO>cf9GUs%U4qD8L_a zTIX@yRCUYJmeH5lx~cZ2a#ij3X_>W5_Ie|pZx3z9_t#aKc05vrA8&r)WyYJC9s&we zK^HnxWxwb3Ixo0-{H?7@^*sY(UB2gng_8^k8h_os?D(gy$nRD2wy*1kx9x29V)8S4 z!S3?iUA1G#6^dX5J@RtHUL5N+gVrF>r0r4|=t{F5t1sbqufMy982+29#vq)jrhipU zXR0(=SIs^}+QG~Je)V$y>Se}W{&4y7r^}bO!3^5*HORMVd7|k&q(#r?moo;RhIf5a z4u1!W+peii>xg;d%_Zp#?~=R>p*Br~y#Ku8honxstsDW`vmMJzrpiSzesnJ}UGY5z zlHo35@2ir}b<=!pwa#0rHH560V1+j`(aU1E$lLB1%b*OPKETwm+07(dvhjc?IsRg4 zb0ZpCtRU4tij~h#M_GXt2YvpdmGU<5hCsRFCKiNUzY$QewunFh}nq7JM&h1n~VINQV7g zl3_nUv#K#QKf-fh8yq@YUUtupC#3>(-iRK7`P$mw2P2@hC&C@^EP*{?--h(lGJk4( zM!TkRBS3T#;O_P{ERE-Qu;n7ShR0y(n6~NjAI$NXFpR1D8#4~lwSDyIdD(Mac*uE? zu~7Goh)Y+7JxK8WAf^fmV;Z*+`KonauHD=5D5tGsG*#%upAJYkQ91o zv#W+VPE{#_Ss~B{BSo`BhRi}u-bj1CWPiD07q;#SjzZHd78*FW7ceaf(0|YdEXr0) zNg>P^ua`ngwTi0F+p2Gh11&j1MhNJ1e5>{p^ zdr7q}sx(WweAiVwu_+;D$4x{T1Qx=eSR7+xtASelvg*h@`WDo%}y zHo+WSER-0wP#ut6secJEekZ%|R!~k6t7DLBi(`dBj7$YyQ~V6}HJiVBx$I!j)_R z|CQ_>wlRii2-Xx?6srqTPI2lrEVnKd9;rDP&S}0R9Wf4yXMa5!Eet2LmJ{KhR=n;Q zO)$<#K#eb~hR*^g~Ud*POv65Q$sD(d2J;K%E|mp)Jhy&$3laF@gnABvic@ zuz=)9urgvJ;kRfU7U}*zOKF`i=_kUaCGLPS2zPJ{XlT)A{JA02o-IHH*pFIV0WKc5 z@Lj4_hoXYz0Z@wQ_1|QagxEn2mbp0KB=)}n|5ife}jAKzNq4{eBPFcMu(E^_7Ou4F!8iYM5n}DI_&zu6iuF3AY#lDr}CHtNjjV=N3a)v( zYVw+V)+Jjw#DaHKQB;q-0Z%hl0sEI<@neqBo|2beOYwfo>Zn=!-~UFsy0mW$=jE zIi{g1A?ZP>+0Z7d+zoEB8EtsO+5*@2m+?h7NfxwFI&IW?38ZtSiRz{Wc>}iiE`6vP zFv`=(os@QR2^qKMmb)J2+_{{TVH+s+)6CwC-ioaSmi8CQRvNttB!HAo#9BH!Dv9qJ zP=9G~-!c*)N-gKHu^20;!N%4Ev-NlW zNJA#aMzdp*6UK79%bqtIh`ZMp5sn{0%((87s=QLfn|!l%3bxhbRZ#V??0*3aRW)j? zukkBqx-MuIS~hy3w{h2GAT~|#n17>sQ2L_8vkYhL7XG43l;M@e5c3>rZVvCTjl+rT z=adb4o$MNiuIe;Dse@TOo|65PyVP~yBSh6HPE(##BnYoslU7+8A84QQ9h>#=*3>3uPe>MUR0(GkMT9Ct9Ih zE$DSsmllaQ?QO$3`v`OOH_XECT-bru8cL?qPkDQFF;=h1n}0@UhM1NnpJq+B zqQ^vNiIv>c@srHSOSRb+<)%#yct1NakHl!(;SCV8RFFb^I*n8!PD>O>HU-Q}U^|qg zBw~5N-j;3OaCU(~0-k{1T&u(N3@DbMjBW852x_X-O z1(^|7ahUCEg$nUnAnbFmSdjx?8!E?QROgn( z;tN;IQ8zX0^2}xQ1#;FtKC))ACQTb>(WchwbMS%E+Kf{0Mt>|%$Fo|951gddS|a_# ziT=$JI=N)omYMBe6^&#ae9A|cRXYpBS()+@Kj!6b%1gM^2Yx}&`Fh(PkOhu3RY69S zo$Chd1vw*DZkg<%^yWV~7Qmtig3}sv%)0HePI91acyfG_xk8*u73o|oAHB@5w(rLbuRplwL9{SV3EdzhlK}Fe zUIAgZRIAIdg;^r^k`vw{6;xV^vGqN0F^{Rc{j(PgwMOpx74FE@(x;E{Z%V*5 zgxu+%o^Ar-p;K0IF=ugf&p*|m2{fBF#*=zA#_cYz@2dA8!P}14gAwW!UxAQqQ7G3^ zCLhGh4pk$sae*x*poMvtBmm)yb;N2gUqq02J#x!pfx43cM-f?(fRb;Z~~ zesvodihmf=>j$@yQIRV%tlPw}0By{J$4ygh@RAfXdF`>m@p`#99bU-&lisZm!XZJ- zy(>ui%9Sg5?~c{u78el%{o&m!x+J8p-HThz^cIWac3*eG0HsT|Kh&nEHsLyuGC^2N zN0{?LW9I^u{oz`r4I{S9<+UlpsJIF<96{cwl7H9F@J`mqggw5+4DlWugs38TVO@o; z1!4?RerPP-{u!AQekNo_6vFW?)?Pivzh|k>&LI~aq^-j4wE`#;F~JC}2uOqYnV15& zw($5gCT{KFXY>l8o>I|HtidrkOm@oGBib_hvX1?M86@c@&C6*GKek$OU`=IuAp9H^ z$A9tJsn3L1@H1j@ps;PfLKm^_i->J1)L-BN&$;O$_j!6Ndxir!&mEU!gy#7rD)c={ zL$Bi{Y7b0mKoD*;=D6xeQn9TvbXv?K*PWVZ%tr?{KXF2?JxS=joDb;%FK41(34}j0TeAy7QldEl2L0i2Q z)5d(LdWI1^1uXak@QFXeVkI=?7r%{3d9TQj5M-} zcJ*!gfR|xEI}(1P@nSZ1kA$%DI`~IbY+&FK?m4ATcldkDVOR{OU^3vyS|lXxG;u=0|+oDLkJ(JX#k{O z5(qMW-){iYt^-dIF4<>E4@=uoPsuaIv9cke(RcQhn*jdR%g#eX;APCb~Q zz+F)%I-*w@RE@a|Kr2IihY5Cj;p{X!y(aTA*?nY9&9;1(_0gRUUoMz7eVQHD$N|^W zGkzfq(JoXbDFdCG?CF(fqklo4K~?O80Y@K8&!I2wD_Dy9Vzqp|K$ zzwfacd2HEZj;Oi??KB&)iX&0}UA{|Hib|`s$Ee^h5UX<9}-|#k(GWLBx9q zV!IC5)ACRAnJ1u8QP_N09i$(CpTY&G*$m26zGE}=a8ULZ_BQ9F+xxQ>A$Ig=yS~RL z%6*j`A}uv_M#j^OPPLq&f1uKEdap86#V(~SS?#dvs_KSx-%)9(c{n4_3<3omBM4!jmVZ72*IlSCH|~#kWZCtocLnhE$nmu?gv(NQCGJMN!X%>Kr#sH* z*w4Up${wh<$>_ytf|&A;YCLWM0beUlYY=2 zktywad^hE`>VGqQIZ>@npoB}m{vYtD!Hhoc^U)sijA?45xB}^*BS6QS|4v2pTOz=7 zfM0-Fe<_L}vgH{uAJ9&N)N}CZmt)h@ap?*Ar{Up3AVLoxs=o>#c13z4kIqcQUs!Qp zaU1zVNy8pMFh^X2YbZfulI+10lSCp$^{0#ut-K^8UQeWl=QwRP4|Dc6`z`vq$xRN) zIS^AF78gr-Am}E42AFh6*{ian5MazWwb24Z6j>ZyGT`6EVet@Y01Bz zGRB53LZ9^X-YXS6VZrK1k~u{OTU(nq4CLUNq?o_?#%h_5DFl?+L9Y<5IDDcfxO2Hc zr!vr`h(-5dOr8g`pUNePgF|2%;f4`TByc2(zo?>sJUSq&P)gZN(T-+8J0$Z?UlR-E zs-JaNJ+y6O>y6!-=Mi(}Gm~oflDuFD96h{X^wZ+V>HOzt>6(9~j&spu`U^8WJhhtE z*6YV+3AZxelRX@p1-3x@ou-rB94CL!OOv8700iLs{1%u2Bt#E$KmsDZzz2vI3KUEb zLLi_FzkX}V+FcJ;-@oc^RyFNc6cI|=DEeeS0>`lev~QKT^big{sFK@Z-WC?H2T4J9 z1R?G6Qv)KOa+D9nTou&G2m57WM&42onI6l>_H-AE(SBYAB5;hf^V03D1nYklbAr=s zgm;~F5TyUoxhl(;Uigc2A38(JQnuEi-A7xupM-YrcFOT;mLfQ6b)wnVxgW0605+wTR{{V~eM)6a=nE1)hU^qa zDn>zb%L8|Yu{Y6o=Y^%UPEdch3x`_;Xc2(ApsJ&%!Osn0?Tb<`FInLvT?oQqg$Z{v zN1jf1<51LOmIb5@DfRm9Jd(K}PbB5JN|Zi-b0NGH3C*gnmATiEzSsX<)zjR)TQ%{u zskbux1x&;%E>iQ@&SuT3vfQ&39+yVBt^@^2@6o_E&E;S5qGb1cgCHnBa(^(eQD@Tr zM{*K-F^!ku10^#LJ-j`2fFz8^@4>BUY_>=cKK=l+ryffI3Wx`48(0$n098tp{2wwL z97VNZ$p8QV4gmlF9RL6T0000000000q=B~^lRg_ClW8C-0=pcOJ{uR4a2y3JP)h*< z6ay3h000O8001FKwm|!xrU3u|cLD$a8vp7yqL)Zv1b>zfOX7$k z34Yjd3Zss)}}&iw68J zPU}3bo2qVE+A{huTQ}9-JX}@#eOhKMlfB-E@7E7)$M=_2nRYx zUa*^dcT?>cN`)#|L65xLuos7V&43ydnzUUC16^p*WA!2Y?)7(P5yO9TMGVTBBK@Nx zovGGjT{Zg@c?U25`_;?+tCtyj`NR3kpUz)i2Q#4KYtV1g@WF#c%_Zv%?~=R>;c1!(egA&PACfxlwsHiZXFHacOtp(%m{ot;c1hXqZ+UY_8Zyiyg%Pw;76uk6JA>C? zOE3`cwJjYHTbfEC+}!2eLvojAyh?7H>K;P@*%i83O2W7v=Ga}tg6}1d06za6*|5Jz zHtYu=s~Yp(m)*1DNv!~%H=;*iKDPGf!3gN>i3mqLOJGmfw;}y+IW>QN zN57_aqd*K25bpLhER7dl7= zW1;RFk(90ud(hzRgP1B9j3I6#idE~rT*>JMLGY2(3Y!$85LAKCTaJ3zvk<5wMhsyD zq`!ej(1-??agcbQmpo&)57bBTj@W)Uk0aUT^@!k(Bm4>(|KYFWtPog((V|%*!)Bo-Z?rvMvOip~Guw8BM4{;x3k#gv3z!xKSZIF(9%UP* z-u?#=$^Ho!+uZm@s z=7%}rf1xdXi|7F$i)D3Vwnl4&wubnnb*+v(Bx2wUusionipy=;pKu_w3d9p>!+iq- zGsmaMP<2OhNNEW(UUVt(62yP5T^OOkE-0$WC9`IuV{gGryB&k>>#1#WocwG^Zjr!B zN6h=GVT_(j_VLO@@fW|xU`@TUF1H9@)5Ub1C`Eq`4 zhdb=@in>Eg8xd95U3>Azxf|w`;DESm5G-^|0404TW-_0<-Q?_GSlxdv=A~LrWUSm& z_L62@RB4uU`L3&WVpBrQj+=;b2yBEwu{lO!i$E=YS#{(deG6e8k-NRK(a>q$a(4Ce z+oq|SF!-fJ@Gb%v0rT#m=E26|<$g&%3S4I^uuk^*eaiG_Y4T!5kwi zv>0j73Q$UE05N_hckq7}PR5@PI9I@`#orHx{!4^4$L4$Mb(Rf=o~yufPyDXo9;8*+XRj9uuqO}k_( z0N%uiIwb%emwAI5{!g%gu(J!y1{`dNh~L#$HPX6ByN;LcX!6}1P^bGb(864QFR5a} zyw;X0q1Ub0=17kOlOi^fP730%NcXo{O6z=1|Ks+*#E34FN@ri@pH@Ym@#lt6dyv>9 zVY^a`D>lpfEqs5Lrq!XUV0i#!AsYJ+iTZIbl!ImNzIK%O-;9gDPwS+s5{mmp1v?^d za_+$r(G~-#Vvb)e-T|Wve0BNEfoAEsbQ8?rjWsuyXt?jC>$pzEZ>F6mHyp-szP zK>SBd6{)d`75y3=GcGrTG}Vr~1P{L!%p;lguj>GD@nkAuVNs9eoii zOzt!P-1G)Q|(t#dmYL_~p+-L6}f|8a`4LU41 z37LN)t03XzK{gLR0Tv6Ht+B>ttH|zyC8?s{m+ghDxfw^2)wgWD2yP@^!5wpeVUR$S z>*B17QBKoQv}0Y!A!@s3O~pQa91+Zou-Q6pcwT1wQ#ce(i!u=OXp42$|FGREQx&d+ zJz8U;a*NpT&pA*%Sr(bC4_ID;+f_TTJF+k%LhC+^wX6GbFjO9d^J#T+Dkaw@oA{;-0oN?VHRe7P9H~D7kG;FK;i=gUZ z*#j7=YSdU?6IV_QT|gFEK6+%dao1#^HqG!*e52=J^hJrad1vhw{-R5i;kCw)^E}kt z97Z9HBZ=(iv<-S49U6zB>bN+mlUY1&k^NM-)XyMBh)2f+O<8$Q5MJBiQni0$^^}e` zG^g z=YumM=eUkWnmcZ9B`A{04lLJrZh{(kTW0unDUaH03@&JtwaRbfU|QVmv5<%2$H1VO zJUE&Yuh4!h_;pp67Ku2$Y{P#!`v`NjG|a;9T-br$8cLzlPkDQBHuhYzH$*3< zn3g3U=S|n5$3$w054m~9k8&rk)Mk5>TQ)V|{p7+tGNWyWH^9s?K}zxIbWN!^Em0xa z6fiGA>`=0jh~)))Tef|}*%>AYc%plCsSb-XynKhgi?v3F|6-(IEc|~-+%C9)bpCsw zeb`rh3mmZ+wv158;NST9Nv8zrF31rF51RkNWxiC0k6Ju&T|^X1txZLCCMk5MLpMzh zxY4QR>RHMcR7PCGVRoz)Cj1xPBG8H9$%*)+WlyJEZ;)3Aw^}an| z3leFrf{kc9H4NAbazd`$GTB4f&0je-z@i9-(-w2cyX|v%?@oV%B&iw~42~TtMuVqT zMQiwQh=nH@sAAmkvAy*VpL&sYQUdM6ljD=l72;H;NatdCdH70*SW~PjHN0-@Dz9ZN z?)eZcl+!{tzqmwZ0WW{dur7%a0cgw@PnxFM;9VUc zdF_e8@p7{`%U#Ibj&Ak~<&Yu8BNH68pL%HwtB3wO>S2RGV-eNL3*=rE|#nfVgWO#xEv`30i^-`{IjhYc=xP zD<-_&Vk3X9y$qRgvQ~BP4DVfy=*7}J5Ptml^T&}Xc{&itEugI`Wx7t<}(9&&d zx>tXS$^ApsGfZU(x#Ks$5&SM+@tD0S^3{(0`(OWqQT5;d`d2YQ6}aIea`??THCB7b)_499k-ll{Pw>UoRL z*Y=>s*uOExeoeov3eJaPilEYh)nNR<+76f)uf56XDK)Z0yyqN|g|G}{Yj*PH1AJj_?l-9C*WdX=poGciZt02Oa( zRIfv1$vNU3qhN>b?HL-`^0jVz!pez7gEV1bw6Yvr-Lm{XvZiKxzAO6ZPKS5xr(K_B z$2D@m_4J%yC_~VN$|YqEag{y2@_aOie4!+$9m}r1G^zS66Ddo?6`t-W5l4U1x>}`+ zz+yCwJQ(*qQ6o>RXe@Ozi)BGOEk>+TG}M2W?-Et3(Pr%lD&z~qug%euAu0p7pl9EH z&T9xPAkViTbB+feBUPZ8st}p%&{NbP(HxqX>hbgn`S0lFFZC?9OC-D z*qj6QwEW$C=BZqa#?#D1t7>-`J!usg=;V!#;0Zaw zYOQ#2n44NG>y{WzN%4>tIKE?}$%SZM5l9`$@vI1E14hJP0uX6zr0f#~I)odInM$Gm z35cAs68eK7rG3Zks@zt6hCjwrIuDd`>Fxgkq8b?KLqQ(^qRf9-u|t$AqW1sd_AY4{$e&E{^-{%rqZ zT2v&*4vSQW#o0Jg9&5NYGFi*4McD+jg9j=QDL*UEz&|w5Z2G+b^Zy4>O9KQH00008 z0000X0KgdY$>0D001E<>&l@9um61V912GVV@A(yTROp7?R;Y%ySlr&KEDF63+c6#7 zBsIxa_TM`d(R0Wp%=g~#^72d>kC?GKP)r=Wfw^wf3pz4z4l%30d9CHBx6wI_lYvek zoU#8vOYp_{1g)%rHh9dUhap(*)ppFM5Dn}c(*PIraHI$aBL9mja>#>W0n!46T?9SS z6jAp{xH=A{>1sgz$aYEt0l8+{d8vP)Xt`j2ZV*kz`KGgXK>U-En$oyB@h9m%bf%n*Y6U?t z&SmxpNBMMR-bfTU^r{LexmUOpavpOai%(aqso67>vZby#58|&MTIf91opd-`R6qy$ zc9fm-bRRC$KxwF4)&L;|{zx<~swX7uhb)Q-%yH0M%3wXob5GMZ$;_5pH@LNbGnd;} z)MBLQ0K)N)!IP$7?UPcU1wRt~qF?kyx}Mw;YNqsHUvj(c($OO^m}CnRSh`vdMk>^l zZ$}W{zd9AX6^bgWKNi+j$NEuonZm6_u zM1Z96Ycy~yYyK;qr0ia$j?!9g0A`z(SYtXmM zm5%{<`wz1xA4>rW?v^b^!x8`hK}eH)ATk`l81u>C0000B0ssIV00000000000001_ zfg&4|T_G2f;UF>sJsgu=As3S@Ar%~7x!IYd0RRAZ0ssIT00000000000001_0m>Yc NT_G(7ryc+R001Ojvl{>a diff --git a/src/framework/processing/py/poetry.lock b/src/framework/processing/py/poetry.lock index 969dd9d..de0b541 100644 --- a/src/framework/processing/py/poetry.lock +++ b/src/framework/processing/py/poetry.lock @@ -1,29 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. - -[[package]] -name = "certifi" -version = "2022.9.24" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.6" -files = [ - {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, - {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, -] - -[[package]] -name = "charset-normalizer" -version = "2.1.1" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.6.0" -files = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] - -[package.extras] -unicode-backport = ["unicodedata2"] +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "colorama" @@ -50,37 +25,6 @@ files = [ [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] - -[[package]] -name = "importlib-metadata" -version = "6.7.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, - {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, -] - -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] - [[package]] name = "iniconfig" version = "2.0.0" @@ -92,6 +36,47 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "numpy" +version = "1.26.0" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = "<3.13,>=3.9" +files = [ + {file = "numpy-1.26.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd"}, + {file = "numpy-1.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292"}, + {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68"}, + {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be"}, + {file = "numpy-1.26.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3"}, + {file = "numpy-1.26.0-cp310-cp310-win32.whl", hash = "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896"}, + {file = "numpy-1.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91"}, + {file = "numpy-1.26.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a"}, + {file = "numpy-1.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd"}, + {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208"}, + {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c"}, + {file = "numpy-1.26.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148"}, + {file = "numpy-1.26.0-cp311-cp311-win32.whl", hash = "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229"}, + {file = "numpy-1.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99"}, + {file = "numpy-1.26.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388"}, + {file = "numpy-1.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581"}, + {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb"}, + {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505"}, + {file = "numpy-1.26.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69"}, + {file = "numpy-1.26.0-cp312-cp312-win32.whl", hash = "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95"}, + {file = "numpy-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112"}, + {file = "numpy-1.26.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2"}, + {file = "numpy-1.26.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8"}, + {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f"}, + {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c"}, + {file = "numpy-1.26.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49"}, + {file = "numpy-1.26.0-cp39-cp39-win32.whl", hash = "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b"}, + {file = "numpy-1.26.0-cp39-cp39-win_amd64.whl", hash = "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299"}, + {file = "numpy-1.26.0.tar.gz", hash = "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"}, +] + [[package]] name = "packaging" version = "23.1" @@ -104,18 +89,68 @@ files = [ ] [[package]] -name = "panda" -version = "0.3.1" -description = "A Python implementation of the Panda REST interface" -optional = false -python-versions = "*" -files = [ - {file = "panda-0.3.1.tar.gz", hash = "sha256:f213b848f09268b3e9fce0e103155ab003217c0e27f6048b6194e7f90bb2b716"}, +name = "pandas" +version = "2.1.1" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, + {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, + {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e"}, + {file = "pandas-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2"}, + {file = "pandas-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0"}, + {file = "pandas-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa"}, + {file = "pandas-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98"}, + {file = "pandas-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97"}, + {file = "pandas-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2"}, + {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, ] [package.dependencies] -requests = "*" -setuptools = "*" +numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] [[package]] name = "pluggy" @@ -128,9 +163,6 @@ files = [ {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, ] -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} - [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] @@ -149,7 +181,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" @@ -159,41 +190,40 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] -name = "requests" -version = "2.28.1" -description = "Python HTTP for Humans." +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" optional = false -python-versions = ">=3.7, <4" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] [package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +six = ">=1.5" [[package]] -name = "setuptools" -version = "68.0.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" +name = "pytz" +version = "2023.3.post1" +description = "World timezone definitions, modern and historical" optional = false -python-versions = ">=3.7" +python-versions = "*" files = [ - {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, - {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] [[package]] name = "tomli" @@ -207,48 +237,17 @@ files = [ ] [[package]] -name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +name = "tzdata" +version = "2023.3" +description = "Provider of IANA time zone data" optional = false -python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, -] - -[[package]] -name = "urllib3" -version = "1.26.12" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" -files = [ - {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, - {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "zipp" -version = "3.15.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.7" +python-versions = ">=2" files = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, + {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, + {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - [metadata] lock-version = "2.0" -python-versions = "^3.7" -content-hash = "4e87005a84a3f69b738ac0b93da24a30b5203cdc8c63102f7eee3d96792822a9" +python-versions = "3.10.2" +content-hash = "48fb13f9bc3d436376cbffdbb3a4205a786a9e3b06f794b76f4dfdd2e7779074" diff --git a/src/framework/processing/py/port/script.py b/src/framework/processing/py/port/script.py index 1a5454e..ba6a343 100644 --- a/src/framework/processing/py/port/script.py +++ b/src/framework/processing/py/port/script.py @@ -132,7 +132,7 @@ def load_tiktok_data(json_file): return data -def get_json_data(zip_file): +def get_json_data_from_zip(zip_file): with zipfile.ZipFile(zip_file, "r") as zip: for name in zip.namelist(): if not name.endswith(".json"): @@ -142,6 +142,14 @@ def get_json_data(zip_file): return [load_tiktok_data(json_file)] return [] +def get_json_data_from_file(file_): + # TikTok exports can be a single JSON file or a zipped JSON file + try: + with open(file_) as f: + return [load_tiktok_data(f)] + except (json.decoder.JSONDecodeError, UnicodeDecodeError): + return get_json_data_from_zip(file_) + def filtered_count(data, *key_path): items = get_list(data, *key_path) @@ -400,8 +408,7 @@ def extract_tiktok_data(zip_file): extract_comment_activity, extract_videos_liked, ] - for data in get_json_data(zip_file): - print(repr(data)) + for data in get_json_data_from_file(zip_file): return [ table for table in (extractor(data) for extractor in extractors) @@ -413,9 +420,11 @@ def extract_tiktok_data(zip_file): # Data donation flow # ###################### - ExtractionResult = namedtuple("ExtractionResult", ["id", "title", "data_frame"]) +class InvalidFileError(Exception): + """Indicates that the file does not match expectations.""" + class SkipToNextStep(Exception): pass @@ -431,20 +440,23 @@ def __init__(self, platform, mime_types, extractor, session_id): self.meta_data = [] def process(self): - print("START") with suppress(SkipToNextStep): while True: file_result = yield from self.prompt_file() self.log(f"extracting file") try: - print(file_result) extraction_result = self.extract_data(file_result.value) except IOError as e: - print("IOERROR") self.log(f"prompt confirmation to retry file selection") yield from self.prompt_retry() return + except InvalidFileError: + self.log(f"invalid file detected, prompting for retry") + if (yield from self.prompt_retry()): + continue + else: + return else: if extraction_result is None: try_again = yield from self.prompt_retry() @@ -520,7 +532,7 @@ def __call__(self, session_id): tik_tok_data_donation = DataDonation( - "TikTok", "application/zip, text/plain", extract_tiktok_data + "TikTok", "application/zip, text/plain, application/json", extract_tiktok_data ) @@ -547,8 +559,8 @@ def render_donation_page(platform, body, progress): def retry_confirmation(platform): text = props.Translatable( { - "en": f"Unfortunately, we cannot process your {platform} file. Continue, if you are sure that you selected the right file. Try again to select a different file.", - "nl": f"Helaas, kunnen we uw {platform} bestand niet verwerken. Weet u zeker dat u het juiste bestand heeft gekozen? Ga dan verder. Probeer opnieuw als u een ander bestand wilt kiezen.", + "en": "Unfortunately, we cannot process your data. Please make sure that you selected JSON as a file format when downloading your data from TikTok.", + "nl": "Helaas kunnen we uw gegevens niet verwerken. Zorg ervoor dat u JSON heeft geselecteerd als bestandsformaat bij het downloaden van uw gegevens van TikTok.", } ) ok = props.Translatable({"en": "Try again", "nl": "Probeer opnieuw"}) diff --git a/src/framework/processing/py/pyproject.toml b/src/framework/processing/py/pyproject.toml index 71cd9e5..1e37728 100644 --- a/src/framework/processing/py/pyproject.toml +++ b/src/framework/processing/py/pyproject.toml @@ -5,8 +5,8 @@ description = "Port package with Data Donation logic" authors = ["Emiel van der Veen "] [tool.poetry.dependencies] -python = "^3.7" -panda = "^0.3.1" +python = "3.10.2" +pandas = "^2.1.1" [tool.poetry.group.test.dependencies] pytest = "^7.4.2" diff --git a/src/framework/processing/py/tests/tiktok_test.py b/src/framework/processing/py/tests/tiktok_test.py index aa1c423..ae6dab7 100644 --- a/src/framework/processing/py/tests/tiktok_test.py +++ b/src/framework/processing/py/tests/tiktok_test.py @@ -10,6 +10,7 @@ import json import zipfile import io +import tempfile from pathlib import Path from dataclasses import dataclass from inspect import cleandoc @@ -19,6 +20,92 @@ from port import script +complete_contents = { + "Profile": { + "Profile Information": { + "ProfileMap": {"userName": "jane_doe", "likesReceived": "77"} + } + }, + "Direct Messages": { + "Chat History": { + "ChatHistory": { + "Chat History with john_doe:": [ + { + "Date": "2023-01-08 17:38:59", + "From": "john_doe", + "Content": "https://www.tiktokv.com/share/video/7167866677751860486/", + }, + { + "Date": "2023-01-08 17:38:59", + "From": "jane_doe", + "Content": "👍", + }, + { + "Date": "2023-01-08 18:12:45", + "From": "john_doe", + "Content": "cool", + }, + { + "Date": "2023-01-08 18:12:55", + "From": "john_doe", + "Content": "https://www.tiktokv.com/share/video/7175594838077787434/", + }, + ] + } + } + }, + "Activity": { + "Follower List": {"FansList": [{"Date": "2023-01-14 18:01:16"}]}, + "Following List": { + "Following": [ + {"Date": "2023-01-14 18:01:16"}, + {"Date": "2023-01-14 18:02:16"}, + ] + }, + "Like List": { + "ItemFavoriteList": [ + {"Date": "2023-01-14 18:01:16"}, + {"Date": "2023-01-14 18:02:16"}, + ] + }, + "Video Browsing History": { + "VideoList": [ + {"Date": "2023-01-14 18:01:16"}, + {"Date": "2023-01-14 18:02:16"}, + {"Date": "2023-01-14 18:03:16"}, + {"Date": "2023-01-14 18:04:16"}, + ] + }, + }, + "Video": { + "Videos": { + "VideoList": [ + { + "Likes": "1", + + "Date": "2023-01-14 18:01:16"}, + { + "Likes": "1", + + "Date": "2023-01-14 18:02:16"}, + { + "Likes": "1", + + "Date": "2023-01-14 18:03:16"}, + ] + } + }, + "Comment": { + "Comments": { + "CommentList": [ + {"Date": "2023-01-14 18:01:16"}, + {"Date": "2023-01-14 18:02:16"}, + {"Date": "2023-01-14 18:03:16"}, + ] + } + }, +} + def get_test_file(name): return str(Path(__file__).parent.joinpath(name)) @@ -96,20 +183,8 @@ def test_wrong_file_type_is_handled(): "ok": {"translations": {"en": "Try again", "nl": "Probeer opnieuw"}}, "text": { "translations": { - "en": "Unfortunately, we cannot " - "process your TikTok file. " - "Continue, if you are sure " - "that you selected the " - "right file. Try again to " - "select a different file.", - "nl": "Helaas, kunnen we uw " - "TikTok bestand niet " - "verwerken. Weet u zeker " - "dat u het juiste bestand " - "heeft gekozen? Ga dan " - "verder. Probeer opnieuw " - "als u een ander bestand " - "wilt kiezen.", + "en": "Unfortunately, we cannot process your data. Please make sure that you selected JSON as a file format when downloading your data from TikTok.", + "nl": "Helaas kunnen we uw gegevens niet verwerken. Zorg ervoor dat u JSON heeft geselecteerd als bestandsformaat bij het downloaden van uw gegevens van TikTok.", } }, }, @@ -460,12 +535,13 @@ def test_direct_messages_table(): assert "Direct Message Activity" == result.title.translations["en"] reference = """ - Anonymous ID Sent - 0 2 2023-01-08 17:38:59 - 1 1 2023-01-08 17:38:59 - 2 2 2023-01-08 18:12:45 - 3 2 2023-01-08 18:12:55 + Anonymous ID Sent + 0 2 2023-01-08 17:38 + 1 1 2023-01-08 17:38 + 2 2 2023-01-08 18:12 + 3 2 2023-01-08 18:12 """ + print(result.data_frame) assert_frame_str_equal(reference, result.data_frame) @@ -488,12 +564,12 @@ def test_comment_activity_table(): assert "Comment Activity" == result.title.translations["en"] reference = """ - Posted on - 0 2023-03-26 15:40:06 - 1 2023-03-18 12:52:35 - 2 2023-03-11 15:06:35 - 3 2023-03-11 15:05:52 - 4 2023-03-03 14:22:03 + Posted on + 0 2023-03-26 15:40 + 1 2023-03-18 12:52 + 2 2023-03-11 15:06 + 3 2023-03-11 15:05 + 4 2023-03-03 14:22 """ assert_frame_str_equal(reference, result.data_frame) @@ -525,15 +601,16 @@ def test_videos_liked_table(): } result = script.extract_videos_liked(data) assert "tiktok_videos_liked" == result.id - assert "Comment Activity" == result.title.translations["en"] + assert "Videos liked" == result.title.translations["en"] reference = """ - Liked Link - 0 2023-03-26 15:39:28 https://www.tiktokv.com/share/video/7199666315... - 1 2023-03-18 12:53:14 https://www.tiktokv.com/share/video/7209355519... - 2 2023-03-18 12:53:11 https://www.tiktokv.com/share/video/7209700824... - 3 2023-03-11 15:06:37 https://www.tiktokv.com/share/video/7191669641... + Liked Link + 0 2023-03-26 15:39 https://www.tiktokv.com/share/video/7199666315... + 1 2023-03-18 12:53 https://www.tiktokv.com/share/video/7209355519... + 2 2023-03-18 12:53 https://www.tiktokv.com/share/video/7209700824... + 3 2023-03-11 15:06 https://www.tiktokv.com/share/video/7191669641... """ + print(result.data_frame) assert_frame_str_equal(reference, result.data_frame) @@ -541,29 +618,45 @@ def test_timezone_to_uk(): assert False -def test_get_json_data_with_invalid_json(): +def test_get_json_data_from_zip_with_invalid_json(): f = make_zip({"test.json": "testing"}) - assert [] == script.get_json_data(f) + assert [] == script.get_json_data_from_zip(f) -def test_get_json_data_with_non_tiktok_json(): +def test_get_json_data_from_zip_with_non_tiktok_json(): f = make_zip({"test.json": "{}"}) - assert [] == script.get_json_data(f) + assert [] == script.get_json_data_from_zip(f) -def test_get_json_data_with_valid_tiktok_json(): +def test_get_json_data_from_zip_with_valid_tiktok_json(): tiktok_data = { "Profile": {"Profile Information": {"ProfileMap": {"userName": "test"}}} } f = make_zip({"test.json": json.dumps(tiktok_data)}) - assert [tiktok_data] == script.get_json_data(f) + assert [tiktok_data] == script.get_json_data_from_zip(f) + + +def test_extract_tiktok_data_works_with_zip_files(): + with tempfile.NamedTemporaryFile() as f: + make_zip({"test.json": json.dumps(complete_contents)}, f) + f.flush() + result = script.extract_tiktok_data(f.name) + assert len(result) > 1 + +def test_extract_tiktok_data_works_with_json_files(): + with tempfile.NamedTemporaryFile(mode="w+t") as f: + json.dump(complete_contents, f) + f.flush() + result = script.extract_tiktok_data(f.name) + assert len(result) > 1 -def make_zip(contents): - f = io.BytesIO() - z = zipfile.ZipFile(f, mode="w") +def make_zip(contents, out=None): + if out is None: + out = io.BytesIO() + z = zipfile.ZipFile(out, mode="w") for filename, data in contents.items(): z.writestr(filename, data) z.close() - f.seek(0) - return f + out.seek(0) + return out