diff --git a/ofscraper/api/subscriptions.py b/ofscraper/api/subscriptions.py index 10df7852a..9f7cad704 100644 --- a/ofscraper/api/subscriptions.py +++ b/ofscraper/api/subscriptions.py @@ -58,6 +58,8 @@ async def activeHelper(subscribe_count,c): funct=scrape_subscriptions_active tasks = [asyncio.create_task(funct(c,offset)) for offset in range(0, subscribe_count+1, 10)] + tasks.extend([asyncio.create_task(funct(c,subscribe_count+1,recur=True))] ) + new_tasks=[] while tasks: done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) @@ -88,6 +90,8 @@ async def expiredHelper(subscribe_count,c): funct=scrape_subscriptions_disabled tasks = [asyncio.create_task(funct(c,offset)) for offset in range(0, subscribe_count+1, 10)] + tasks.extend([asyncio.create_task(funct(c,subscribe_count+1,recur=True))] ) + new_tasks=[] while tasks: done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) @@ -114,16 +118,17 @@ async def expiredHelper(subscribe_count,c): @retry(retry=retry_if_not_exception_type(KeyboardInterrupt),stop=stop_after_attempt(constants.NUM_TRIES),wait=wait_random(min=constants.OF_MIN, max=constants.OF_MAX),reraise=True) -async def scrape_subscriptions_active(c,offset=0,num=0) -> list: +async def scrape_subscriptions_active(c,offset=0,num=0,recur=False) -> list: async with c.requests( subscriptionsActiveEP.format(offset))() as r: if r.ok: subscriptions = (await r.json_())["list"] + log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}") if len(subscriptions)==0: return subscriptions - log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}") - if num+len(subscriptions)<10: - new_tasks.append(asyncio.create_task(scrape_subscriptions_active(c,offset=offset+len(subscriptions),num= num+len(subscriptions)))) - + elif recur==False: + None + elif (await r.json_())["hasMore"]==True: + new_tasks.append(asyncio.create_task(scrape_subscriptions_active(c,recur=True,offset=offset+len(subscriptions)))) return subscriptions else: log.debug(f"[bold]subscriptions response status code:[/bold]{r.status}") @@ -133,38 +138,25 @@ async def scrape_subscriptions_active(c,offset=0,num=0) -> list: @retry(retry=retry_if_not_exception_type(KeyboardInterrupt),stop=stop_after_attempt(constants.NUM_TRIES),wait=wait_random(min=constants.OF_MIN, max=constants.OF_MAX),reraise=True) -async def scrape_subscriptions_disabled(c,offset=0,num=0) -> list: +async def scrape_subscriptions_disabled(c,offset=0,num=0,recur=False) -> list: async with c.requests( subscriptionsExpiredEP.format(offset))() as r: if r.ok: subscriptions = (await r.json_())["list"] + log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}") if len(subscriptions)==0: return subscriptions - log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}") - if num+len(subscriptions)<10: - new_tasks.append(asyncio.create_task(scrape_subscriptions_disabled(c,offset=offset+len(subscriptions),num= num+len(subscriptions)))) - + elif recur==False: + None + elif (await r.json_())["hasMore"]==True: + new_tasks.append(asyncio.create_task(scrape_subscriptions_disabled(c,recur=True,offset=offset+len(subscriptions)))) + return subscriptions else: log.debug(f"[bold]subscriptions response status code:[/bold]{r.status}") log.debug(f"[bold]subscriptions response:[/bold] {await r.text_()}") log.debug(f"[bold]subscriptions headers:[/bold] {r.headers}") r.raise_for_status() - -@retry(retry=retry_if_not_exception_type(KeyboardInterrupt),stop=stop_after_attempt(constants.NUM_TRIES),wait=wait_random(min=constants.OF_MIN, max=constants.OF_MAX),reraise=True) -async def sort_list(c) -> list: - async with c.requests( constants.sortSubscriptions,method="post",json={ - "order": "users.name", - "direction": "desc", - "type": "all" -})() as r: - if r.ok: - None - else: - log.debug(f"[bold]subscriptions response status code:[/bold]{r.status}") - log.debug(f"[bold]subscriptions response:[/bold] {await r.text_()}") - log.debug(f"[bold]subscriptions headers:[/bold] {r.headers}") - r.raise_for_status() - + def parse_subscriptions(subscriptions: list) -> list: datenow=arrow.now() data = [ @@ -176,7 +168,9 @@ def parse_subscriptions(subscriptions: list) -> list: "expired":profile.get("subscribedByData").get("expiredAt") if profile.get("subscribedByData") else None, "subscribed":(profile.get("subscribedByData").get("subscribes") or [{}])[0].get("startDate") if profile.get("subscribedByData") else None , "renewed":profile.get("subscribedByData").get("renewedAt") if profile.get("subscribedByData") else None, - "active" : arrow.get(profile.get("subscribedByData").get("expiredAt"))>datenow if profile.get("subscribedByData") else None + "active" : arrow.get(profile.get("subscribedByExpireDate") or (profile.get("subscribedByData",{}) or {}).get("expiredAt") or 0)>datenow + or (profile.get("subscribedByData",{}) or {}).get("status")=="Set to Expire" + } for profile in subscriptions] @@ -194,4 +188,3 @@ def setpricehelper(data): -