From 8af2fa9e78c7ad3ff58636678d1e7ee0dd8787c2 Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Tue, 10 Sep 2024 13:49:13 -0400 Subject: [PATCH 01/12] allow next year's sessions to be scraped at end of current --- lametro/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lametro/__init__.py b/lametro/__init__.py index 400e825..240d442 100644 --- a/lametro/__init__.py +++ b/lametro/__init__.py @@ -3,6 +3,7 @@ from .bills import LametroBillScraper from .people import LametroPersonScraper from .events import LametroEventScraper +from datetime import datetime class Lametro(Jurisdiction): @@ -17,7 +18,14 @@ class Lametro(Jurisdiction): } legislative_sessions = [] - for year in range(2014, 2025): + + today = datetime.now() + this_year = today.year + allowed_years = list(range(2014, this_year)) + if (today.month == 6 and today.day >= 23) or today.month >= 7: + allowed_years.append(this_year) + + for year in allowed_years: session = { "identifier": "{}".format(year), "start_date": "{}-07-01".format(year), From 980e9221d5fb3ea30014f98e4ccfb7ea4bb85b3a Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Tue, 10 Sep 2024 13:50:11 -0400 Subject: [PATCH 02/12] include test for next year's sessions --- requirements.txt | 1 + tests/test_jurisdiction.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/test_jurisdiction.py diff --git a/requirements.txt b/requirements.txt index 557e0e6..35cbca1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ pytest-mock==1.10.1 requests-mock==1.5.2 requests[security] sentry-sdk +freezegun==1.5.1 diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py new file mode 100644 index 0000000..3878bab --- /dev/null +++ b/tests/test_jurisdiction.py @@ -0,0 +1,23 @@ +from datetime import datetime + +import pytest +from freezegun import freeze_time + + +@freeze_time('2024-06-23') +def test_legislative_session(mocker): + ''' + Test that next fiscal year's legislative sessions are included + when it's the last week of the current one. + ''' + fake_now = datetime.now() + mocker.patch( + "lametro.Lametro.today", return_value=fake_now + ) + from lametro import Lametro + + latest_session_date = Lametro.legislative_sessions[-1]["end_date"] + next_year = str(fake_now.year + 1) + + assert next_year in latest_session_date + breakpoint() From 428f6353bfbfac2f9983bc90a4016c72fd84eb6d Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Tue, 10 Sep 2024 14:13:18 -0400 Subject: [PATCH 03/12] docker command --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index edc295e..5d47485 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,4 +15,4 @@ jobs: steps: - uses: actions/checkout@v1 - name: Build containers and run tests - run: docker-compose -f docker-compose.yml -f tests/docker-compose.yml run --rm scrapers + run: docker compose -f docker-compose.yml -f tests/docker-compose.yml run --rm scrapers From 148b222a8bbd3c9de1c45732b21b710f1c51228c Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Tue, 10 Sep 2024 14:16:57 -0400 Subject: [PATCH 04/12] remove rogue breakpoint --- tests/test_jurisdiction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py index 3878bab..e18ef86 100644 --- a/tests/test_jurisdiction.py +++ b/tests/test_jurisdiction.py @@ -20,4 +20,3 @@ def test_legislative_session(mocker): next_year = str(fake_now.year + 1) assert next_year in latest_session_date - breakpoint() From 94d1a832018d46627bcec9de5abab581aafe033d Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Wed, 11 Sep 2024 16:33:18 -0400 Subject: [PATCH 05/12] calculate legislative sessions on the fly --- lametro/__init__.py | 34 +++++++++++++++++++--------------- lametro/bills.py | 2 +- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lametro/__init__.py b/lametro/__init__.py index 240d442..2fa8327 100644 --- a/lametro/__init__.py +++ b/lametro/__init__.py @@ -17,21 +17,25 @@ class Lametro(Jurisdiction): "events": LametroEventScraper, } - legislative_sessions = [] - - today = datetime.now() - this_year = today.year - allowed_years = list(range(2014, this_year)) - if (today.month == 6 and today.day >= 23) or today.month >= 7: - allowed_years.append(this_year) - - for year in allowed_years: - session = { - "identifier": "{}".format(year), - "start_date": "{}-07-01".format(year), - "end_date": "{}-06-30".format(year + 1), - } - legislative_sessions.append(session) + @staticmethod + def get_legislative_sessions(): + sessions = [] + today = datetime.now() + this_year = today.year + allowed_years = list(range(2014, this_year)) + + if (today.month == 6 and today.day >= 23) or today.month >= 7: + allowed_years.append(this_year) + + for year in allowed_years: + session = { + "identifier": "{}".format(year), + "start_date": "{}-07-01".format(year), + "end_date": "{}-06-30".format(year + 1), + } + sessions.append(session) + + return sessions def get_organizations(self): org = Organization(name="Board of Directors", classification="legislature") diff --git a/lametro/bills.py b/lametro/bills.py index b674df7..bdf22ad 100644 --- a/lametro/bills.py +++ b/lametro/bills.py @@ -76,7 +76,7 @@ def session(self, action_date) : localize = pytz.timezone(self.TIMEZONE).localize fmt = '%Y-%m-%d' - for session in Lametro.legislative_sessions: + for session in Lametro.get_legislative_sessions(): start_datetime = datetime.datetime.strptime(session['start_date'], fmt) end_datetime = datetime.datetime.strptime(session['end_date'], fmt) From 3ffc12068d1fadb6618d62a1b0d32fd22cc3bd15 Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Wed, 11 Sep 2024 16:33:57 -0400 Subject: [PATCH 06/12] get test working properly --- tests/test_jurisdiction.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py index e18ef86..1c2d13a 100644 --- a/tests/test_jurisdiction.py +++ b/tests/test_jurisdiction.py @@ -2,21 +2,33 @@ import pytest from freezegun import freeze_time +from lametro import Lametro -@freeze_time('2024-06-23') -def test_legislative_session(mocker): +@pytest.mark.parametrize("test_date", [ + '2024-06-10', + '2024-06-23', + '2024-10-01', + '2025-05-01', + '2025-06-30' +]) +def test_legislative_session(test_date): ''' Test that next fiscal year's legislative sessions are included - when it's the last week of the current one. + only when it's the last week of the current one. ''' - fake_now = datetime.now() - mocker.patch( - "lametro.Lametro.today", return_value=fake_now - ) - from lametro import Lametro + date_format = "%Y-%m-%d" + test_year = test_date.split('-')[0] + last_week_of_year = datetime.strptime(f"{test_year}-06-23", date_format).date() - latest_session_date = Lametro.legislative_sessions[-1]["end_date"] - next_year = str(fake_now.year + 1) + with freeze_time(test_date): + fake_now = datetime.now() + fake_date = fake_now.date() - assert next_year in latest_session_date + latest_session_date = Lametro.get_legislative_sessions()[-1]["end_date"] + next_year = str(fake_now.year + 1) + + if fake_date < last_week_of_year: + assert next_year not in latest_session_date + elif fake_date >= last_week_of_year: + assert next_year in latest_session_date From 36403a770d981100d1bd84249eeefd2e9f61606e Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Wed, 11 Sep 2024 16:40:02 -0400 Subject: [PATCH 07/12] yield legislative sessions instead of building list --- lametro/__init__.py | 4 +--- tests/test_jurisdiction.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lametro/__init__.py b/lametro/__init__.py index 2fa8327..5123f02 100644 --- a/lametro/__init__.py +++ b/lametro/__init__.py @@ -19,7 +19,6 @@ class Lametro(Jurisdiction): @staticmethod def get_legislative_sessions(): - sessions = [] today = datetime.now() this_year = today.year allowed_years = list(range(2014, this_year)) @@ -33,9 +32,8 @@ def get_legislative_sessions(): "start_date": "{}-07-01".format(year), "end_date": "{}-06-30".format(year + 1), } - sessions.append(session) + yield session - return sessions def get_organizations(self): org = Organization(name="Board of Directors", classification="legislature") diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py index 1c2d13a..a99ce7b 100644 --- a/tests/test_jurisdiction.py +++ b/tests/test_jurisdiction.py @@ -24,9 +24,10 @@ def test_legislative_session(test_date): with freeze_time(test_date): fake_now = datetime.now() fake_date = fake_now.date() + next_year = str(fake_now.year + 1) - latest_session_date = Lametro.get_legislative_sessions()[-1]["end_date"] - next_year = str(fake_now.year + 1) + sessions = list(Lametro.get_legislative_sessions()) + latest_session_date = sessions[-1]["end_date"] if fake_date < last_week_of_year: assert next_year not in latest_session_date From 1025a1cff27103382d5c54cae701ea87e7b051d5 Mon Sep 17 00:00:00 2001 From: Hannah Cushman Garland Date: Thu, 12 Sep 2024 10:04:49 -0500 Subject: [PATCH 08/12] Re-encrypted keys --- keyrings/live/blackbox-admins.txt | 1 + keyrings/live/pubring.gpg | Bin 8378 -> 10700 bytes lametro/secrets.py.gpg | Bin 1423 -> 1954 bytes 3 files changed, 1 insertion(+) diff --git a/keyrings/live/blackbox-admins.txt b/keyrings/live/blackbox-admins.txt index e0dcd53..b3926c8 100644 --- a/keyrings/live/blackbox-admins.txt +++ b/keyrings/live/blackbox-admins.txt @@ -1,3 +1,4 @@ datamade@scrapers-us-municipal fgregg@datamade.us hannah@datamade.us +xavier.medrano@datamade.us diff --git a/keyrings/live/pubring.gpg b/keyrings/live/pubring.gpg index b693412da61421909cf39e81fa5e3aa332993d73..f7f2987465a68b7dc4a4effbeb35d49ef4aea01c 100644 GIT binary patch delta 2325 zcmajfYdq5n0|)T`Hg{sR(cET{lHx(K0_7@QtAG3W60s$6K?sYvdPMr=^%BV^z3Ut4bD)7zLoMXAcAIO}c~#dD6M^N%HZ#>li1M$zwU5le_6 zN7s!5@af zdE=91Uqe_$id42~!0I|iP8eEy#c793TbhW3C(b<$_bj=cFNn9W$CFirEY{?jnf|1P z;b^e?AS|Ty!wCBj<(ES`%nv10n|I?7+egmG>oGs8G7^=#@Fw?N~gps-5B@DUI`j~ z^;4of<^tt1)jK!@qfh&*t6%-QXXq-(f(cL;J_JMvFp)aK=cFMMN~DEb z&|4kaAQo?*OKEGZ@I(J|GN>$!Ul`Wu3-F5y3kgD@ickSb5n&-wD4!G*27#glp@(k) zBEs;0R*ODxpmAT!Kl;onXU*Ql=c`b8aKMF9r0EwOWm&I}Z+n7mF|V~W6th562J~=V zZwx66H34yNMjYWKWy+?TiiWCTt(YG4S97ZAe4}?5`h#H)N&Opi**KuYS`PS-LWRNn zlea;_xt!+7F{cSh!{Z}25q&>S;aQsRrMV%oy522m8ecCq%ZH3_8LbqjsUQh5P18+x z*X|J#m|L`gfN+%=puVjGwEf)gewAC{q2+p8;cb0wpb4>hIh|Pi2zby2-`H5ATOTS+ zFEb9LMmV}qTL!xauAxGbKS`R^M1f&Be%z!?0dj1r3IE{svkjKkClOnvX6ljrPN|ng z*QB}qIIcV=fS!upHB;{-mTv9{YqK+|RwC(DO{C=Bsp%J3Is(*gSqqmRw5vEK0q#Ca*xyu%pv(uOn-`GAb z?uTnw3@k?MgL^A#j~f(ElBfEl@h_|tkLl{cHah}4myqiX3BhD&i6~O|R@-FkRZP1h zW7|#K)I6Q`_|RC!_l7x_vE0W~@06D3(^!VTzu@Cm;YL?peg7DH1<|FmfU8^HKVtW$ zRsewgc}Sr7zgUgqDc`XwG-RGr(Lw)t@KH@s^5@5ItQSmQ*)v8ke!6naSp+M0O?i^4lCutDt=`keo9U ze3+}x`7w%pTDfw2D`7P-Tke#Z0<%JiV{1~Z9t7Z?soh&EdL|focd~r(7uAa+S(bN! zn~{@OFBuDNq&>ebsng`PW)}6jsBonLr<$xUN`w=H2=Li--n+h~e^KtVq#TdF;hX1F zsQ5*2ob_cFWlQ0U=A9@}5mGUE2R7i%hh{D5@%-Yr1cO4i^-T2#ONo|L`zQOK<<<)n zb$?RaJ}WHo@?xkTJrA=^37 zrrqRZhp@}-%k4@le*AB$)-yT|`wGqv^P;s_EpvLzFGW4wdgeXlKWWyHCO(~4)4QpI zGH_4`;_48HJE#zg1fA2B4$-Q%8GV=IXR+}h(N;Sm`gur$tB~5%lZRc=SdMZ2@%xLz zcPWa>yvKiGl>pV}LxF$NbL{`nBlb5v|Le<>A8?=z%w}I|G&-~}-A94wtM47HrFwWA zdr;j!&Z}uRIg1|G%{5(;t6LmDs3Vpd+eVa88}a*mr!OfrMJ?~@Q*NBI9WK2eqLn0hr8`1pWC;jQYZG?xs!`=^RycfJdlf-^l79jG7RaI7_H{4nt zdMRqg=YAr~Cd~QXrRz5fmj{pLO?R8mW@(p3%xMml4~88mcqL*vU^UP~TKm90{m2DA z#Av4YiY;(!picciuoo8Et3|#6$tVvf5c2Xo*I8TiloXv=ebO)e!Lt==c5g>ybOVwwujrpW*L6pL$d_MU))BI z_N=2NJ6==3DQTS_TT9nB(SB_t!O+cK$PNjVI zY=PF)8bAxTP7-8|7WJ-~U$YVM?Hah8*OCc{8{^^(dL_2EyFKf)H)W$rMg!O>KXmx%fA7b4nr;g delta 7 OcmX>TyvuRJE(HJ&*8`;h diff --git a/lametro/secrets.py.gpg b/lametro/secrets.py.gpg index fc3b135a84f09683f9a0acd542fef757c4008242..1a281eb18a5af16b17f94f70f206614c0061e9dc 100644 GIT binary patch literal 1954 zcmV;T2VMAu0t^FXIka+BbKVvK5CFvtG-j(rDs15{6LH^J60%m(?H5GwhHeWQVO10Y zeLE&hBC|e8;UyQWN#Je|%Dpxfc>#)G;uFC`i^E5_a1{qaf5YbDcM6+ZaA(FHHZ_^%ozHa`igKV1xfh`&9=z8B;I zt)*+X`Y&ZaCJ`Nu^HM|sSL7+XcX;Vo6Pg@1KwTqEL}d+5{`e!&CJM1ns-ieKgO#lg_0+9ZB|^XBn^s4;x)vcbMr9|d zQBe_`-#u&(lWR^}qF=37j-FY7C5=?t=ifiZhh%MgLhzY!?pl$YM{vh6#z~;-UM0#? z>dBz}G7UxE)N$CwO49j{(;ysGpIb4Bb3Cx>%7e>}4=L`boS2Jl>VPAn*ykM~g+>2J zZvGT2(6a6O7JB5h@Eey^1w~ldoDslbpzD$j&D)F7@8Xadp&9Pi`J3)j7vnXxgvVxW z44DHdaw@f>TH3;zvdXB4NBhG`XC*uU;Pfv6?-0`waf;u~g;s&OR=u5bJ1Pc?&T#+2 z={Q;+JOa~lMv)9GWG|O_1MX`_|64oV&sX5J?O*Zn|M^QnQ&RXlJ7W~h z!zE(*43^MT+XoTUBb@rOdW01c_d%pTG5eT);2OI#isLB%UiSDC8iG`iYY*>M07|1{ zIZpSIpb|sUK^U4J{KjSE*DU~g!X!mL=i_8tOYUO|2={(RR#IuP>`!#Z+yKkxTw^z- z4NUmr82z(ReK0VrXRd?;xOE&gkm01*9%dfz!S2qj`VAfa0~!00Ep9xSO-kJR5mi!* z&1Y+Noi~**wWdB9qT#SRyJI)is@D-yi~zoGtHd1!tu#k_P*;!h{pIwJ59?x7nX=6R zq*XI_gkWMiHlIKe67TAYI2$jQIuu1`yrFE9H>RB-!XX}_u*WEt@?AyUVr9feh=tQ- zPs&YQP|hpxX<>Q`V3IEkVkb~Wbwb8XqAqpKbd#E($^t_!9Hf{0e^4#f3>~)Xyh>gj z!shs5cXnspO<%uZM}xk0I=|?{?~K_T%IddkcUjr)2t*S9CVfD@nB}_BMWG4aVboQBl7KVVrU5&;K5unJUY;t%%Qj1e6y)NMCC zR%Vn)`fAwh_k|gCEeLmiFmQU~M&C}-a8p_|TH3tY-W3S(-e2vAB}1rgtXr!CE)0?O zWq0W(TJGIu(Q^&}fi>FMshYD2`2 z7BH<=Jj4ti9yj=kRR)AZ?P;|J2G5-G2vN?7v$@P;xnJ-}T}*i4Kp06Z53(RQX`rJ& z|q93U&yl9`E5Kjj{jN%T!QTPns*{Z#r&X~-j=g6 z-#?YEhQAc314w##<~WuKL=@N>uO&nFb~yRCz;5n`FFMAu@ZLqW!Jc0#S}`U(zei#A z%K0Gl>=tJgCclS7a(Is<0cI|mkt?LD9FAu5gpHv^;KW}pn%>`m>3)$ygXs*NmO_rQ zA|l)S$lFhQ5SGgXQBwF0J_Y>Kzh$qqMb<=!-NHVYEFjad9789c6C>biM-%-3iwz^2B!o z2;@0FU{+-KHS(S4+ZZU4hPN3x(!f#ySk3FZuAT0AFo9R+g70?Up%mj_*>tFNh-)M6 zI=1RxzN>lrah9S?t}PtD~k$Ok1xI;0!N4Lki(Nq4eh zH6p)hRpw2ZFuW=PNkeJZ|EWwFEBn69jK0S=-{adF*SAm-6`8Gk!5C(T(qwTMr4-Io z3{;V^NRJQo{&0*lq>6$gA^uJIcAqMBkw)97YNYei6wF*4sAH8aZ7?xX=K}yu6u^xG z5NNp-xQ6<#<7kN#*edp&@4KI=j7Ne0zvQY`e{L|$k(;E?5^9E5uH)MfGC$QGWi1~f owgASa?D72WLvgJj+Ad!8(MkxY@O;|#*@T7A0Nq8SkWTdYo}v!5oB#j- literal 1423 zcmV;A1#tR>0t^FXIka+BbKVvK5CEsz_FU)RVqBe2HL-NkN4Wy0o2Q)u8=+T@j8tnZ z466{P4=}h@j}FM7Ba=@CH_uyoyw3;e$xaKk<9S!^euK9YvGiA6yI0#nL#U!JNfV4k z%37!b90M2f57aRw(l?#YmnK)m6I-({tZf{49OIeh?SC{!4j18X2#V9a(46v`@4)j| zlg8stVgcegVZGeBS6AD$_5d4;KEt2=MQY#< z>2l0_{&|rDhn){ac(bbUN1pE4d2>3v;{K>ot=f7(9-WWNH9OZ^iP%-&-|}^Bd4J|z za4?}2M~y0t6>|KN>->T0HfKXeD6|Xw=E-a70;K`J6EU(j$oVF&hm9HlFMnrh$KXt6 zhH2?0RPj^u#|-bXeH10Fb>zgiYA{|@yQRZN^)}ovhuG@x-C1M89AnMjA;$^xVMi0Y zkiB!ZfB|~#*`D?S!hMAT3`0Dy24StZlI#W)BY71Scw=`-wpBQ>gV zhnhWXj{K^f;&A>>>#!?s8cV8EIxMQsg2D{sL6b#;k3!09)>9SNmXe;UsLpogTb|g0 z@&@v}5LA?-YU9PN+eXQag4rM_3JKNlaW7NtYkz4G>GI{k)hZKskXg1=gKjFhCTNFx zc`EKI+i$0ddA%=J9cM+)L(&IXH?PEsY~ksK0+`Z<-`YVJYIM=115q?B*u$~-)z>LA zYfMBzq2J%WR#*iObG5~v?WFuE?o(x1EG--tyZ0?Po~bQluG+Wb1mN zp9VLH7f>Y=%w+g}CnnGTI~|MB!xSBWAV#{Q{?Ll+8H|<@wxODxEclutBh& z#t$^PC?qdad7G#em(UN=<*1h@FbWZp;tv*;c*L?pIC5v<+CO=jpl5FsoBkYFr%GOd zQcRhALVA-D;7uoU#Y4fg_OvE(!jX@vOjG9_u|TRKKRt5UOK*GKP#h-DNo;xal+cF2 zV(xqhqV6+Ax}WD}5d~(kWuLyI^cOoTE25L|+lZv6e5ctw2U$6KOJKZITboQBl7KVVrU5&;K5Mcrlfq20q0>Z@d>5%BC? zN-OO7pd97(@zomAKwk+~FjxVa^D|BI2?D#!q91tJrk<~j|s5iF*s-@dVbeagXK_ME>z*JWR;z~ z9D~-siso%P66;$xK|H%hE_nc}lj5 Date: Tue, 17 Sep 2024 09:56:57 -0400 Subject: [PATCH 09/12] maintain signature from base Jurisdiction class --- lametro/__init__.py | 4 ++-- lametro/bills.py | 2 +- tests/test_jurisdiction.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lametro/__init__.py b/lametro/__init__.py index 5123f02..c791272 100644 --- a/lametro/__init__.py +++ b/lametro/__init__.py @@ -17,8 +17,8 @@ class Lametro(Jurisdiction): "events": LametroEventScraper, } - @staticmethod - def get_legislative_sessions(): + @property + def legislative_sessions(): today = datetime.now() this_year = today.year allowed_years = list(range(2014, this_year)) diff --git a/lametro/bills.py b/lametro/bills.py index bdf22ad..0bddda6 100644 --- a/lametro/bills.py +++ b/lametro/bills.py @@ -76,7 +76,7 @@ def session(self, action_date) : localize = pytz.timezone(self.TIMEZONE).localize fmt = '%Y-%m-%d' - for session in Lametro.get_legislative_sessions(): + for session in Lametro.legislative_sessions.fget(): start_datetime = datetime.datetime.strptime(session['start_date'], fmt) end_datetime = datetime.datetime.strptime(session['end_date'], fmt) diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py index a99ce7b..8582dfe 100644 --- a/tests/test_jurisdiction.py +++ b/tests/test_jurisdiction.py @@ -12,7 +12,7 @@ '2025-05-01', '2025-06-30' ]) -def test_legislative_session(test_date): +def test_legislative_sessions(test_date): ''' Test that next fiscal year's legislative sessions are included only when it's the last week of the current one. @@ -26,7 +26,7 @@ def test_legislative_session(test_date): fake_date = fake_now.date() next_year = str(fake_now.year + 1) - sessions = list(Lametro.get_legislative_sessions()) + sessions = list(Lametro.legislative_sessions.fget()) latest_session_date = sessions[-1]["end_date"] if fake_date < last_week_of_year: From f39f93ab0b20bc40411822935a6b1cb6aa051808 Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Tue, 17 Sep 2024 10:57:27 -0400 Subject: [PATCH 10/12] add comments. add test parameter for last week of year --- lametro/__init__.py | 6 ++++++ tests/test_jurisdiction.py | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lametro/__init__.py b/lametro/__init__.py index c791272..bc3f84a 100644 --- a/lametro/__init__.py +++ b/lametro/__init__.py @@ -19,11 +19,17 @@ class Lametro(Jurisdiction): @property def legislative_sessions(): + ''' + Yield each year that we'd like to scrape today. + Allow for the next fiscal year to be scraped during + and after the last week of the current fiscal year. + ''' today = datetime.now() this_year = today.year allowed_years = list(range(2014, this_year)) if (today.month == 6 and today.day >= 23) or today.month >= 7: + # The last week of this fiscal year has begun. Start scraping the next year allowed_years.append(this_year) for year in allowed_years: diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py index 8582dfe..eb0fd6e 100644 --- a/tests/test_jurisdiction.py +++ b/tests/test_jurisdiction.py @@ -6,11 +6,12 @@ @pytest.mark.parametrize("test_date", [ - '2024-06-10', - '2024-06-23', - '2024-10-01', - '2025-05-01', - '2025-06-30' + '2024-06-10', # before the last week of the fiscal year + '2024-06-23', # start of the last week of the fiscal year + '2024-10-01', # well after the last week + '2024-12-24', # last week of regular year + '2025-05-01', # before the last week of the next fiscal year + '2025-06-30' # end of the last week of the next fiscal year ]) def test_legislative_sessions(test_date): ''' From e705f7faac5d7ddc1cd6d5980216cdd246664a24 Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Wed, 18 Sep 2024 11:00:12 -0400 Subject: [PATCH 11/12] get legislative_sessions back into a list --- lametro/__init__.py | 5 ++++- tests/test_jurisdiction.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lametro/__init__.py b/lametro/__init__.py index bc3f84a..fd805c1 100644 --- a/lametro/__init__.py +++ b/lametro/__init__.py @@ -27,6 +27,7 @@ def legislative_sessions(): today = datetime.now() this_year = today.year allowed_years = list(range(2014, this_year)) + sessions = [] if (today.month == 6 and today.day >= 23) or today.month >= 7: # The last week of this fiscal year has begun. Start scraping the next year @@ -38,7 +39,9 @@ def legislative_sessions(): "start_date": "{}-07-01".format(year), "end_date": "{}-06-30".format(year + 1), } - yield session + sessions.append(session) + + return sessions def get_organizations(self): diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py index eb0fd6e..adc01d2 100644 --- a/tests/test_jurisdiction.py +++ b/tests/test_jurisdiction.py @@ -27,7 +27,7 @@ def test_legislative_sessions(test_date): fake_date = fake_now.date() next_year = str(fake_now.year + 1) - sessions = list(Lametro.legislative_sessions.fget()) + sessions = Lametro.legislative_sessions.fget() latest_session_date = sessions[-1]["end_date"] if fake_date < last_week_of_year: From 1df5f4f332683c897a1b9b6405760aa2483b091f Mon Sep 17 00:00:00 2001 From: Xavier Medrano Date: Tue, 24 Sep 2024 10:34:16 -0400 Subject: [PATCH 12/12] require an instance for legislative_sessions --- lametro/__init__.py | 2 +- lametro/bills.py | 2 +- tests/test_jurisdiction.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lametro/__init__.py b/lametro/__init__.py index fd805c1..ab8e110 100644 --- a/lametro/__init__.py +++ b/lametro/__init__.py @@ -18,7 +18,7 @@ class Lametro(Jurisdiction): } @property - def legislative_sessions(): + def legislative_sessions(self): ''' Yield each year that we'd like to scrape today. Allow for the next fiscal year to be scraped during diff --git a/lametro/bills.py b/lametro/bills.py index 0bddda6..5ae24dd 100644 --- a/lametro/bills.py +++ b/lametro/bills.py @@ -76,7 +76,7 @@ def session(self, action_date) : localize = pytz.timezone(self.TIMEZONE).localize fmt = '%Y-%m-%d' - for session in Lametro.legislative_sessions.fget(): + for session in Lametro().legislative_sessions: start_datetime = datetime.datetime.strptime(session['start_date'], fmt) end_datetime = datetime.datetime.strptime(session['end_date'], fmt) diff --git a/tests/test_jurisdiction.py b/tests/test_jurisdiction.py index adc01d2..aacc801 100644 --- a/tests/test_jurisdiction.py +++ b/tests/test_jurisdiction.py @@ -25,9 +25,9 @@ def test_legislative_sessions(test_date): with freeze_time(test_date): fake_now = datetime.now() fake_date = fake_now.date() - next_year = str(fake_now.year + 1) + next_year = str(fake_now.year + 1) - sessions = Lametro.legislative_sessions.fget() + sessions = Lametro().legislative_sessions latest_session_date = sessions[-1]["end_date"] if fake_date < last_week_of_year: