fifo gather

dblp_crawler/gather.py

@@ -0,0 +1,29 @@
+import asyncio
+
+
+async def gather(*coros_or_futures):
+    queue = asyncio.Queue()
+
+    async def task(coro_or_future):
+        result = await coro_or_future
+        await queue.put(result)
+
+    tasks = [task(c) for c in coros_or_futures]
+    task_sum = asyncio.gather(*tasks)
+    for _ in range(len(tasks)):
+        yield await queue.get()
+    await task_sum
+
+if __name__ == "__main__":
+    import random
+
+    async def main():
+        async def wait(n):
+            await asyncio.sleep(n)
+            return n
+        tasks = [wait(n) for n in range(10)]
+        random.shuffle(tasks)
+        async for n in gather(*tasks):
+            print(n)
+
+    asyncio.run(main())

dblp_crawler/graph.py

@@ -4,6 +4,7 @@
 from typing import Optional, Iterable, AsyncIterable, List
 
 from dblp_crawler import download_person, DBLPPerson, Publication, download_journal_list, JournalList
+from .gather import gather
 
 logger = logging.getLogger("graph")
 
@@ -44,7 +45,7 @@ async def init_persons_from_journals(self) -> None:
                     atasks.append(self._collect_authors(publication))  # 遍历这个文章
                     logger.debug(str(publication))
             asc, afc = 0, 0
-            for publication, success, fail in await asyncio.gather(*atasks):
+            async for publication, success, fail in gather(*atasks):
                 asc += success
                 afc += fail
                 yield publication
@@ -117,7 +118,7 @@ async def _bfs_once(self):
             total_publication_count += publication_count
 
         # 执行任务：获取已有作者列表中的论文
-        for publication, success, fail in await asyncio.gather(*atasks):
+        async for publication, success, fail in gather(*atasks):
             total_author_succ_count += success
             total_author_fail_count += fail
             yield publication

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='dblp_crawler',
-    version='2.0.1',
+    version='2.1',
     author='yindaheng98',
     author_email='[email protected]',
     url='https://github.com/yindaheng98/dblp-crawler',