From f48af73dcd8b1fa142094506d7ac731ce4a12397 Mon Sep 17 00:00:00 2001 From: arunkannawadi Date: Wed, 4 Jan 2023 17:08:29 -0500 Subject: [PATCH 1/2] Use a default user agent to circumvent fake-useragent errors --- scholarly/_proxy_generator.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scholarly/_proxy_generator.py b/scholarly/_proxy_generator.py index 676b28a..f451ec5 100644 --- a/scholarly/_proxy_generator.py +++ b/scholarly/_proxy_generator.py @@ -14,7 +14,6 @@ from selenium.common.exceptions import WebDriverException, UnexpectedAlertPresentException from selenium.webdriver.firefox.options import Options as FirefoxOptions from urllib.parse import urlparse -from fake_useragent import UserAgent from contextlib import contextmanager from deprecated import deprecated try: @@ -24,6 +23,13 @@ except ImportError: stem = None +try: + from fake_useragent import UserAgent + FAKE_USERAGENT = True +except Exception: + FAKE_USERAGENT = False + DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36' + from .data_types import ProxyMode @@ -454,13 +460,18 @@ def _new_session(self, **kwargs): # self._session = httpx.Client() self.got_403 = False - # Suppress the misleading traceback from UserAgent() - with self._suppress_logger('fake_useragent'): - _HEADERS = { - 'accept-language': 'en-US,en', - 'accept': 'text/html,application/xhtml+xml,application/xml', - 'User-Agent': UserAgent().random, - } + if FAKE_USERAGENT: + # Suppress the misleading traceback from UserAgent() + with self._suppress_logger('fake_useragent'): + user_agent = UserAgent().random + else: + user_agent = DEFAULT_USER_AGENT + + _HEADERS = { + 'accept-language': 'en-US,en', + 'accept': 'text/html,application/xhtml+xml,application/xml', + 'User-Agent': user_agent, + } # self._session.headers.update(_HEADERS) init_kwargs.update(headers=_HEADERS) From 434b25cedd3019f21373fa75796a213e21cd4d35 Mon Sep 17 00:00:00 2001 From: arunkannawadi Date: Wed, 4 Jan 2023 17:10:17 -0500 Subject: [PATCH 2/2] Bump version to 1.7.10-alpha --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b974fcf..586a071 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='scholarly', - version='1.7.9', + version='1.7.10-alpha', author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva, Arun Kannawadi', author_email='steven@cholewiak.com, panos@stern.nyu.edu, vsilva@ualberta.ca, arunkannawadi@astro.princeton.edu', description='Simple access to Google Scholar authors and citations',