Make Twisted[http2] installation optional (scrapy#5113)

Co-authored-by: Eugenio Lacuesta <[email protected]>

conftest.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pytest
+from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
 
@@ -25,6 +26,14 @@ def _py_files(folder):
     if file_path and file_path[0] != '#':
         collect_ignore.append(file_path)
 
+if not H2_ENABLED:
+    collect_ignore.extend(
+        (
+            'scrapy/core/downloader/handlers/http2.py',
+            *_py_files("scrapy/core/http2"),
+        )
+    )
+
 
 @pytest.fixture()
 def chdir(tmpdir):

docs/topics/settings.rst

@@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``::
 
 .. _http2:
 
-The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
-:setting:`DOWNLOAD_HANDLERS` as follows::
+The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
 
-    DOWNLOAD_HANDLERS = {
-        'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
-    }
+#.  Install ``Twisted[http2]>=17.9.0`` to install the packages required to
+    enable HTTP/2 support in Twisted.
+
+#.  Update :setting:`DOWNLOAD_HANDLERS` as follows::
+
+        DOWNLOAD_HANDLERS = {
+            'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
+        }
 
 .. warning::
 

setup.py

@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
 
 
 install_requires = [
-    'Twisted[http2]>=17.9.0',
+    'Twisted>=17.9.0',
     'cryptography>=2.0',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
@@ -31,7 +31,6 @@ def has_environment_marker_platform_impl_support():
     'zope.interface>=4.1.3',
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
-    'h2>=3.0,<4.0',
     'setuptools',
 ]
 extras_require = {}

tests/test_downloader_handlers_http2.py

@@ -1,14 +1,14 @@
 import json
-from unittest import mock
+from unittest import mock, skipIf
 
 from pytest import mark
 from testfixtures import LogCapture
 from twisted.internet import defer, error, reactor
 from twisted.trial import unittest
 from twisted.web import server
 from twisted.web.error import SchemeNotSupported
+from twisted.web.http import H2_ENABLED
 
-from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.misc import create_instance
@@ -21,11 +21,17 @@
 )
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2TestCase(Https11TestCase):
+
     scheme = 'https'
-    download_handler_cls = H2DownloadHandler
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
+
     def test_protocol(self):
         request = Request(self.getURL("host"), method="GET")
         d = self.download_request(request, Spider("foo"))
@@ -187,9 +193,14 @@ def setUp(self):
         super(Https2InvalidDNSPattern, self).setUp()
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2CustomCiphers(Https11CustomCiphers):
     scheme = 'https'
-    download_handler_cls = H2DownloadHandler
+
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
 
 
 class Http2MockServerTestCase(Http11MockServerTestCase):
@@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase):
     }
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ProxyTestCase(Http11ProxyTestCase):
     # only used for HTTPS tests
     keyfile = 'keys/localhost.key'
@@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
     scheme = 'https'
     host = u'127.0.0.1'
 
-    download_handler_cls = H2DownloadHandler
     expected_http_proxy_request_body = b'/'
 
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
+
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
         self.port = reactor.listenSSL(

tests/test_http2_client_protocol.py

@@ -5,10 +5,9 @@
 import shutil
 import string
 from ipaddress import IPv4Address
-from unittest import mock
+from unittest import mock, skipIf
 from urllib.parse import urlencode
 
-from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
@@ -17,12 +16,10 @@
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 from twisted.web.client import ResponseFailed, URI
-from twisted.web.http import Request as TxRequest
+from twisted.web.http import H2_ENABLED, Request as TxRequest
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
-from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
     return PrivateCertificate.loadPEM(pem)
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ClientProtocolTestCase(TestCase):
     scheme = 'https'
     key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
@@ -220,6 +218,7 @@ def setUp(self):
         uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
 
         self.conn_closed_deferred = Deferred()
+        from scrapy.core.http2.protocol import H2ClientFactory
         h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
@@ -426,6 +425,7 @@ def test_received_dataloss_response(self):
 
         def assert_failure(failure: Failure):
             self.assertTrue(len(failure.value.reasons) > 0)
+            from h2.exceptions import InvalidBodyLengthError
             self.assertTrue(any(
                 isinstance(error, InvalidBodyLengthError)
                 for error in failure.value.reasons
@@ -511,6 +511,7 @@ def test_inactive_stream(self):
 
         def assert_inactive_stream(failure):
             self.assertIsNotNone(failure.check(ResponseFailed))
+            from scrapy.core.http2.stream import InactiveStreamClosed
             self.assertTrue(any(
                 isinstance(e, InactiveStreamClosed)
                 for e in failure.value.reasons
@@ -596,6 +597,7 @@ def _check_invalid_netloc(self, url):
         request = Request(url)
 
         def assert_invalid_hostname(failure: Failure):
+            from scrapy.core.http2.stream import InvalidHostname
             self.assertIsNotNone(failure.check(InvalidHostname))
             error_msg = str(failure.value)
             self.assertIn('localhost', error_msg)
@@ -633,6 +635,7 @@ def test_connection_timeout(self):
 
         def assert_timeout_error(failure: Failure):
             for err in failure.value.reasons:
+                from scrapy.core.http2.protocol import H2ClientProtocol
                 if isinstance(err, TimeoutError):
                     self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))
                     break

tox.ini

@@ -50,19 +50,16 @@ commands =
 basepython = python3
 deps =
     {[testenv]deps}
+    # Twisted[http2] is required to import some files
+    Twisted[http2]>=17.9.0
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
 basepython = python3
 deps =
-    {[testenv]deps}
-    # Optional dependencies
-    boto
-    reppy
-    robotexclusionrulesparser
-    # Test dependencies
+    {[testenv:extra-deps]deps}
     pylint
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
@@ -119,9 +116,11 @@ setenv =
 [testenv:extra-deps]
 deps =
     {[testenv]deps}
+    boto
     reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
+    Twisted[http2]>=17.9.0
 
 [testenv:asyncio]
 commands =