add atch for users

datawhores · Nov 29, 2023 · 88769e9 · 88769e9
1 parent 6a3cc40
commit 88769e9
Showing 1 changed file with 21 additions and 28 deletions.
diff --git a/ofscraper/api/subscriptions.py b/ofscraper/api/subscriptions.py
@@ -58,6 +58,8 @@ async def activeHelper(subscribe_count,c):
     funct=scrape_subscriptions_active
 
     tasks = [asyncio.create_task(funct(c,offset)) for offset in  range(0, subscribe_count+1, 10)] 
+    tasks.extend([asyncio.create_task(funct(c,subscribe_count+1,recur=True))] ) 
+
     new_tasks=[]
     while tasks:
         done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) 
@@ -88,6 +90,8 @@ async def expiredHelper(subscribe_count,c):
     funct=scrape_subscriptions_disabled
 
     tasks = [asyncio.create_task(funct(c,offset)) for offset in  range(0, subscribe_count+1, 10)] 
+    tasks.extend([asyncio.create_task(funct(c,subscribe_count+1,recur=True))] ) 
+
     new_tasks=[]
     while tasks:
         done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) 
@@ -114,16 +118,17 @@ async def expiredHelper(subscribe_count,c):
 
 
 @retry(retry=retry_if_not_exception_type(KeyboardInterrupt),stop=stop_after_attempt(constants.NUM_TRIES),wait=wait_random(min=constants.OF_MIN, max=constants.OF_MAX),reraise=True)   
-async def scrape_subscriptions_active(c,offset=0,num=0) -> list:
+async def scrape_subscriptions_active(c,offset=0,num=0,recur=False) -> list:
         async with c.requests( subscriptionsActiveEP.format(offset))() as r:
             if r.ok:
                 subscriptions = (await r.json_())["list"]
+                log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}")      
                 if len(subscriptions)==0:
                      return subscriptions
-                log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}")      
-                if num+len(subscriptions)<10:
-                    new_tasks.append(asyncio.create_task(scrape_subscriptions_active(c,offset=offset+len(subscriptions),num= num+len(subscriptions))))
-
+                elif recur==False:
+                    None
+                elif (await r.json_())["hasMore"]==True:
+                    new_tasks.append(asyncio.create_task(scrape_subscriptions_active(c,recur=True,offset=offset+len(subscriptions))))
                 return subscriptions
             else:
                 log.debug(f"[bold]subscriptions response status code:[/bold]{r.status}")
@@ -133,38 +138,25 @@ async def scrape_subscriptions_active(c,offset=0,num=0) -> list:
 
 
 @retry(retry=retry_if_not_exception_type(KeyboardInterrupt),stop=stop_after_attempt(constants.NUM_TRIES),wait=wait_random(min=constants.OF_MIN, max=constants.OF_MAX),reraise=True)   
-async def scrape_subscriptions_disabled(c,offset=0,num=0) -> list:
+async def scrape_subscriptions_disabled(c,offset=0,num=0,recur=False) -> list:
         async with c.requests( subscriptionsExpiredEP.format(offset))() as r:
             if r.ok:
                 subscriptions = (await r.json_())["list"]
+                log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}") 
                 if len(subscriptions)==0:
                      return subscriptions
-                log.debug(f"usernames offset {offset}: usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}")      
-                if num+len(subscriptions)<10:
-                    new_tasks.append(asyncio.create_task(scrape_subscriptions_disabled(c,offset=offset+len(subscriptions),num= num+len(subscriptions))))
-
+                elif recur==False:
+                    None
+                elif (await r.json_())["hasMore"]==True:
+                    new_tasks.append(asyncio.create_task(scrape_subscriptions_disabled(c,recur=True,offset=offset+len(subscriptions))))
+
                 return subscriptions
             else:
                 log.debug(f"[bold]subscriptions response status code:[/bold]{r.status}")
                 log.debug(f"[bold]subscriptions response:[/bold] {await r.text_()}")
                 log.debug(f"[bold]subscriptions headers:[/bold] {r.headers}")
                 r.raise_for_status()
-
-@retry(retry=retry_if_not_exception_type(KeyboardInterrupt),stop=stop_after_attempt(constants.NUM_TRIES),wait=wait_random(min=constants.OF_MIN, max=constants.OF_MAX),reraise=True)   
-async def sort_list(c) -> list:
-        async with c.requests( constants.sortSubscriptions,method="post",json={
-  "order": "users.name",
-  "direction": "desc",
-  "type": "all"
-})() as r:
-            if r.ok:
-                None
-            else:
-                log.debug(f"[bold]subscriptions response status code:[/bold]{r.status}")
-                log.debug(f"[bold]subscriptions response:[/bold] {await r.text_()}")
-                log.debug(f"[bold]subscriptions headers:[/bold] {r.headers}")
-                r.raise_for_status()
-
+
 def parse_subscriptions(subscriptions: list) -> list:
     datenow=arrow.now()
     data = [
@@ -176,7 +168,9 @@ def parse_subscriptions(subscriptions: list) -> list:
          "expired":profile.get("subscribedByData").get("expiredAt") if profile.get("subscribedByData") else None,
          "subscribed":(profile.get("subscribedByData").get("subscribes") or [{}])[0].get("startDate") if profile.get("subscribedByData") else None ,
          "renewed":profile.get("subscribedByData").get("renewedAt") if profile.get("subscribedByData") else None,
-        "active" :  arrow.get(profile.get("subscribedByData").get("expiredAt"))>datenow if profile.get("subscribedByData") else None
+        "active" :  arrow.get(profile.get("subscribedByExpireDate") or (profile.get("subscribedByData",{}) or {}).get("expiredAt") or 0)>datenow 
+        or (profile.get("subscribedByData",{}) or {}).get("status")=="Set to Expire"
+
 
 
          } for profile in subscriptions]
@@ -194,4 +188,3 @@ def setpricehelper(data):
 
 
 
-