Poetry install performance enhancement for low-resource environments

.gitignore

@@ -42,4 +42,7 @@ poetry/core/*
 
 .pythonlibs
 
-poetry-*-bundle.tgz
+poetry-*-bundle.tgz
+
+poetry_env
+proj

.replit

@@ -4,8 +4,10 @@ modules = ["python-3.10:v16-20230726-64244b3"]
 channel = "stable-22_11"
 
 [env]
-PYTHONPATH = "$PYTHONPATH:$REPL_HOME/src"
-POETRY_INSTALLER_MODERN_INSTALLATION = "0"
+# PYTHONPATH = "$PYTHONPATH:$REPL_HOME/src"
+# POETRY_INSTALLER_PARALLEL = "0"
+POETRY_DOWNLOAD_WITH_CURL = "1"
+POETRY_INSTALLER_MODERN_INSTALLATION = "1"
 POETRY_PIP_FROM_PATH = "1"
 POETRY_PIP_NO_PREFIX = "1"
 POETRY_USE_USER_SITE = "1"

README.md

@@ -113,6 +113,46 @@ external formats like virtual environments
 installation script
 * [website](https://github.com/python-poetry/website): The official Poetry website and blog
 
+## How to test in a Repl
+
+There are 2 ways to test poetry in a Repl.
+
+## Method 1
+
+1. Uncomment `PYTHONPATH = "$PYTHONPATH:$REPL_HOME/src"` in `.replit`
+2. create a test project via:
+    * mkdir proj
+    * cd proj
+    * poetry init # and go through the prompts
+3. Run `python -m poetry ...`
+
+This will install libs into `.pythonlibs`, but has the shortcoming that poetry won't distinguish its
+own dependencies from the test project's.
+
+## Method 2
+
+1. Comment out `PYTHONPATH = "$PYTHONPATH:$REPL_HOME/src"` in `.replit`
+2. create a test project via:
+    * mkdir proj
+    * cd proj
+    * poetry init # and go through the prompts
+3. cd ..
+4. ./install_poetry_in_venv.sh
+5. cd proj
+6. Run `../poetry_env/bin/poetry ...`
+
+This will also install libs into `.pythonlibs`, which will belong solely to the test project. Poetry's
+dependencies live inside poetry_env.
+
+If you want to "reset" the libs you can:
+
+1. cd proj
+2. rm poetry.lock
+3. rm -fr ../.pythonlibs
+4. rm -fr ../.cache/pypoetry
+5. rid pyproject.toml of previously installed libraries
+6. Now you can test installing stuff as if starting from scratch
+
 ## Bundle
 
 For the Replit [Python Nix modules](https://github.com/replit/nixmodules/tree/main/pkgs/modules/python), we build a separate Poetry bundle for each supported version of Python 

install_poetry_in_venv.sh

@@ -0,0 +1,9 @@
+# This script installs poetry within its own venv the way
+# it is when deployed so that its deps are isolated from the deps of the projects it manages
+rm -fr poetry_env
+poetry build
+python -m venv poetry_env
+touch poetry_env/poetry_env
+poetry_env/bin/pip install dist/poetry-1.5.2-py3-none-any.whl
+# inspired by https://stackoverflow.com/a/584926:
+sed -i 's@/usr/bin/env python3@'"$REPL_HOME"'/poetry_env/bin/python3@g' poetry_env/bin/poetry

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "poetry"
-version = "1.5.1"
+version = "1.5.2"
 description = "Python dependency management and packaging made easy."
 authors = ["Sébastien Eustace <[email protected]>"]
 maintainers = [

src/poetry/__version__.py

@@ -16,4 +16,4 @@
 try:
   __version__ = version("poetry")
 except:
-  __version__ = "1.5.1"
+  __version__ = "1.5.2"

src/poetry/installation/executor.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import subprocess
 import contextlib
 import csv
 import itertools
@@ -29,7 +30,7 @@
 from poetry.utils.authenticator import Authenticator
 from poetry.utils.cache import ArtifactCache
 from poetry.utils.env import EnvCommandError
-from poetry.utils.helpers import atomic_open
+from poetry.utils.helpers import atomic_open, download_file_with_curl
 from poetry.utils.helpers import get_file_hash
 from poetry.utils.helpers import pluralize
 from poetry.utils.helpers import remove_directory
@@ -608,7 +609,7 @@ def _traditional_install(self, operation: Install | Update) -> int:
         )
         self._write(operation, message)
         return self.pip_install(req, upgrade=operation.job_type == "update")
-        
+
     def _update(self, operation: Install | Update) -> int:
         return self._install(operation)
 
@@ -783,7 +784,7 @@ def _maybe_add_yanked_warning(self, link: Link, operation: Install | Update) ->
             if link.yanked_reason:
                 message += f" Reason for being yanked: {link.yanked_reason}"
             self._yanked_warnings.append(message)
-                    
+
     def _download(self, operation: Install | Update) -> Path:
         link = self._chooser.choose_for(operation.package)
         self._maybe_add_yanked_warning(link, operation)
@@ -863,6 +864,16 @@ def _validate_archive_hash(archive: Path, package: Package) -> str:
         return archive_hash
 
     def _download_archive(self, operation: Install | Update, link: Link) -> Path:
+        archive = (
+            self._artifact_cache.get_cache_directory_for_link(link) / link.filename
+        )
+        archive.parent.mkdir(parents=True, exist_ok=True)
+
+        url = str(link)
+        if os.getenv("POETRY_DOWNLOAD_WITH_CURL") == "1" and url.startswith("https://files.pythonhosted.org/"):
+            download_file_with_curl(url, str(archive))
+            return archive
+
         response = self._authenticator.request(
             "get", link.url, stream=True, io=self._sections.get(id(operation), self._io)
         )
@@ -889,27 +900,25 @@ def _download_archive(self, operation: Install | Update, link: Link) -> Path:
                 progress.start()
 
         done = 0
-        archive = (
-            self._artifact_cache.get_cache_directory_for_link(link) / link.filename
-        )
-        archive.parent.mkdir(parents=True, exist_ok=True)
+
         with atomic_open(archive) as f:
             for chunk in response.iter_content(chunk_size=4096):
                 if not chunk:
                     break
-
+        
                 done += len(chunk)
-
+        
                 if progress:
                     with self._lock:
                         progress.set_progress(done)
-
+        
                 f.write(chunk)
-
+        
         if progress:
             with self._lock:
                 progress.finish()
 
+
         return archive
 
     def _should_write_operation(self, operation: Operation) -> bool:

src/poetry/installation/wheel_installer.py

@@ -2,6 +2,7 @@
 
 import platform
 import sys
+import os
 
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -40,7 +41,21 @@ def write_to_fs(
         from installer.utils import copyfileobj_with_hashing
         from installer.utils import make_file_executable
 
-        target_path = Path(self.scheme_dict[scheme]) / path
+        if os.getenv("POETRY_USE_USER_SITE") == "1":
+            # this remapping of the base target path allows the modern installer
+            # to write files into the userbase directory (.pythonlibs) instead of the system python
+            # directory which are readonly in a Nix installation
+            if scheme in ["platlib", "purelib"] and "usersite" in self.scheme_dict:
+                target_path = Path(self.scheme_dict["usersite"]) / path
+            elif scheme == "data" and "userbase" in self.scheme_dict:
+                target_path = Path(self.scheme_dict["userbase"]) / path
+            elif scheme == "scripts" and "userbase" in self.scheme_dict:
+                target_path = Path(self.scheme_dict["userbase"] + "/bin") / path
+            else:
+                target_path = Path(self.scheme_dict[scheme]) / path
+        else:
+            target_path = Path(self.scheme_dict[scheme]) / path
+
         if target_path.exists():
             # Contrary to the base library we don't raise an error
             # here since it can break namespace packages (like Poetry's)
@@ -60,6 +75,28 @@ def write_to_fs(
 
         return RecordEntry(path, Hash(self.hash_algorithm, hash_), size)
 
+    # method override to insert custom path mapping logic similar to in
+    # write_to_fs
+    def _path_with_destdir(self, scheme: Scheme, path: str) -> str:
+        if os.getenv("POETRY_USE_USER_SITE") == "1":
+            if scheme in ["platlib", "purelib"] and "usersite" in self.scheme_dict:
+                basepath = self.scheme_dict["usersite"]
+            elif scheme == "data" and "userbase" in self.scheme_dict:
+                basepath = self.scheme_dict["userbase"]
+            elif scheme == "scripts" and "userbase" in self.scheme_dict:
+                basepath = self.scheme_dict["userbase"] + "/bin"
+            else:
+                basepath = self.scheme_dict[scheme]
+        else:
+             basepath = self.scheme_dict[scheme]
+
+        file = os.path.join(basepath, path)
+        if self.destdir is not None:
+            file_path = Path(file)
+            rel_path = file_path.relative_to(file_path.anchor)
+            return os.path.join(self.destdir, rel_path)
+        return file
+
     def for_source(self, source: WheelFile) -> WheelDestination:
         scheme_dict = self.scheme_dict.copy()
 

src/poetry/utils/helpers.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import subprocess
 import hashlib
 import io
 import os
@@ -95,6 +96,10 @@ def download_file(
     session: Authenticator | Session | None = None,
     chunk_size: int = 1024,
 ) -> None:
+    if os.getenv("POETRY_DOWNLOAD_WITH_CURL") == "1" and url.startswith("https://files.pythonhosted.org/"):
+        download_file_with_curl(url, str(dest))
+        return
+
     import requests
 
     from poetry.puzzle.provider import Indicator
@@ -133,6 +138,11 @@ def download_file(
                             last_percent = percent
                             update_context(f"Downloading {url} {percent:3}%")
 
+def download_file_with_curl(
+    url: str,
+    dest: str,
+) -> None:
+    subprocess.run(['curl', url, '--silent', '--output', dest], check=True)
 
 def get_package_version_display_string(
     package: Package, root: Path | None = None