Improve package manager tests (#1754)

Backports a number of updates/improvements to the pip/Poetry/Pipenv
package manager integration tests found while working on the tests for uv.

spec/fixtures/pipenv_editable/.python-version

@@ -1 +1,3 @@
+# Note: This test has to use Python 3.12 until we work around the
+# Pipenv editable VCS dependency cache invalidation bug.
 3.12

spec/fixtures/pipenv_editable/Pipfile

@@ -4,6 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
+gunicorn = {git = "git+https://github.com/benoitc/gunicorn", ref = "20.1.0", editable = true}
 local-package-pyproject-toml = {file = "packages/local_package_pyproject_toml", editable = true}
 local-package-setup-py = {file = "packages/local_package_setup_py", editable = true}
-gunicorn = {git = "git+https://github.com/benoitc/gunicorn", ref = "20.1.0", editable = true}
+pipenv-editable = {file = ".", editable = true}

spec/fixtures/pipenv_editable/pyproject.toml

@@ -0,0 +1,8 @@
+[project]
+name = "pipenv-editable"
+version = "0.0.0"
+requires-python = ">=3.12"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

spec/fixtures/poetry_editable/bin/test-entrypoints.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 cd .heroku/python/lib/python*/site-packages/
 
-# List any path like strings in .pth, and finder files in site-packages.
+# List any path like strings in the .pth and finder files in site-packages.
 grep --extended-regexp --only-matching -- '/\S+' *.pth __editable___*_finder.py | sort
 echo
 

spec/fixtures/poetry_editable/pyproject.toml

@@ -6,7 +6,7 @@ authors = []
 
 [tool.poetry.dependencies]
 python = "^3.13"
-gunicorn = { git = "https://github.com/benoitc/gunicorn.git", tag = "20.1.0", develop = true }
+gunicorn = { git = "https://github.com/benoitc/gunicorn.git", develop = true }
 local-package-pyproject-toml = { path = "packages/local_package_pyproject_toml", develop = true }
 local-package-setup-py = { path = "packages/local_package_setup_py", develop = true }
 

spec/hatchet/ci_spec.rb

@@ -10,7 +10,7 @@
 
     it 'installs both normal and test dependencies and uses cache on subsequent runs' do
       app.run_ci do |test_run|
-        expect(test_run.output).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE))
+        expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE))
           -----> Python app detected
           -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version
           -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
@@ -62,7 +62,7 @@
         REGEX
 
         test_run.run_again
-        expect(test_run.output).to include(<<~OUTPUT)
+        expect(clean_output(test_run.output)).to include(<<~OUTPUT)
           -----> Python app detected
           -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version
           -----> Restoring cache
@@ -81,7 +81,7 @@
 
     it 'installs both normal and test dependencies and uses cache on subsequent runs' do
       app.run_ci do |test_run|
-        expect(test_run.output).to match(Regexp.new(<<~REGEX))
+        expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX))
           -----> Python app detected
           -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version
           -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
@@ -134,7 +134,7 @@
         REGEX
 
         test_run.run_again
-        expect(test_run.output).to match(Regexp.new(<<~REGEX))
+        expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX))
           -----> Python app detected
           -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version
           -----> Restoring cache
@@ -156,7 +156,7 @@
 
     it 'installs both normal and test dependencies and uses cache on subsequent runs' do
       app.run_ci do |test_run|
-        expect(test_run.output).to match(Regexp.new(<<~REGEX))
+        expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX))
           -----> Python app detected
           -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version
           -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
@@ -217,7 +217,7 @@
         REGEX
 
         test_run.run_again
-        expect(test_run.output).to include(<<~OUTPUT)
+        expect(clean_output(test_run.output)).to include(<<~OUTPUT)
           -----> Python app detected
           -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version
           -----> Restoring cache

spec/hatchet/pip_spec.rb

@@ -91,7 +91,8 @@
   end
 
   # This test intentionally uses Python 3.12, so that we test rewriting using older globally installed
-  # setuptools. The Poetry equivalent of this test covers the PEP-517/518 setuptools case.
+  # setuptools (which causes .egg-link files to be created too). The Pipenv and Poetry equivalents of
+  # this test covers the PEP-517/518 setuptools case.
   context 'when requirements.txt contains editable requirements (both VCS and local package)' do
     let(:buildpacks) { [:default, 'heroku-community/inline'] }
     let(:app) { Hatchet::Runner.new('spec/fixtures/pip_editable', buildpacks:) }
@@ -159,6 +160,8 @@
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
         REGEX
+        # Test that the VCS repo checkout was cached correctly.
+        expect(app.output).to include('Updating /app/.heroku/python/src/gunicorn clone (to revision 20.1.0)')
       end
     end
   end

spec/hatchet/pipenv_spec.rb

@@ -47,7 +47,7 @@
           remote: typing_extensions 4.12.2
           remote: virtualenv        .+
           remote: 
-          remote: \\<module 'typing_extensions' from '/app/.heroku/python/lib/python3.13/site-packages/typing_extensions.py'\\>
+          remote: <module 'typing_extensions' from '/app/.heroku/python/lib/python3.13/site-packages/typing_extensions.py'>
         REGEX
         app.commit!
         app.push!
@@ -348,19 +348,24 @@
     end
   end
 
+  # This test has to use Python 3.12 until we work around the Pipenv editable VCS dependency
+  # cache invalidation bug when using pyproject.toml / PEP517 based installs.
   context 'when Pipfile contains editable requirements' do
     let(:buildpacks) { [:default, 'heroku-community/inline'] }
     let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_editable', buildpacks:) }
 
     it 'rewrites .pth, .egg-link and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do
       app.deploy do |app|
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
+          remote: -----> Installing dependencies using 'pipenv install --deploy'
+          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
           remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
           remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
           remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
           remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote:        _pipenv_editable.pth:/tmp/build_.+
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
@@ -371,6 +376,7 @@
           remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
           remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
           remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote: _pipenv_editable.pth:/tmp/build_.+
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
@@ -384,6 +390,7 @@
           __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
           gunicorn.egg-link:/app/.heroku/python/src/gunicorn
           local-package-setup-py.egg-link:/app/packages/local_package_setup_py
+          _pipenv_editable.pth:/app
 
           Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           Running entrypoint for the setup.py-based local package: Hello setup.py!
@@ -394,12 +401,15 @@
         app.commit!
         app.push!
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
+          remote: -----> Installing dependencies using 'pipenv install --deploy'
+          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
           remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
           remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
           remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
           remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote:        _pipenv_editable.pth:/tmp/build_.+
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
@@ -410,6 +420,7 @@
           remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
           remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
           remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote: _pipenv_editable.pth:/tmp/build_.+
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!