HTTP_TIMEOUT

README.md

@@ -49,9 +49,11 @@ python -m dblp_crawler neo4j -h
 usage: __main__.py neo4j [-h] [--auth AUTH] --uri URI
 
 optional arguments:
-  -h, --help   show this help message and exit
-  --auth AUTH  Auth to neo4j database.
-  --uri URI    URI to neo4j database.
+  -h, --help           show this help message and exit
+  --username USERNAME  Auth username to neo4j database.
+  --password PASSWORD  Auth password to neo4j database.
+  --uri URI            URI to neo4j database.
+  --select             Mark keyword-matched publications in database (set selected=true).
 ```
 
 ### Config environment variables
@@ -67,6 +69,8 @@ optional arguments:
   * default: `30`
 * `HTTP_PROXY`
   * Set it `http://your_user:your_password@your_proxy_url:your_proxy_port` if you want to use proxy
+* `HTTP_TIMEOUT`
+  * Timeout for each http request, in seconds
 * `HTTP_CONCORRENT`
   * Concurrent HTTP requests
   * default: `8`

dblp_crawler/downloader.py

@@ -69,7 +69,9 @@ async def download_item(path: str, cache_days: int) -> Optional[ElementTree.Elem
     async with http_sem:
         try:
             async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
-                async with session.get(url, proxy=os.getenv("HTTP_PROXY")) as response:
+                async with session.get(url,
+                                       proxy=os.getenv("HTTP_PROXY"),
+                                       timeout=os.getenv("HTTP_TIMEOUT") or 30) as response:
                     logger.info(" download: %s" % path)
                     html = await response.text()
                     data = ElementTree.fromstring(html)

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='dblp_crawler',
-    version='2.1',
+    version='2.1.1',
     author='yindaheng98',
     author_email='[email protected]',
     url='https://github.com/yindaheng98/dblp-crawler',