From 37c2809ada67398e74ef7a3d25dc44422072a7e7 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 22 Sep 2020 17:06:17 +0200 Subject: [PATCH] locker: port base implementation from poetry --- poetry/core/lock/__init__.py | 8 + poetry/core/lock/categories.py | 11 + poetry/core/lock/locker.py | 397 ++++++++++++++++++++++++++++++++ poetry/core/poetry.py | 17 +- tests/packages/test_locker.py | 398 +++++++++++++++++++++++++++++++++ 5 files changed, 829 insertions(+), 2 deletions(-) create mode 100644 poetry/core/lock/__init__.py create mode 100644 poetry/core/lock/categories.py create mode 100644 poetry/core/lock/locker.py create mode 100644 tests/packages/test_locker.py diff --git a/poetry/core/lock/__init__.py b/poetry/core/lock/__init__.py new file mode 100644 index 000000000..8b25df780 --- /dev/null +++ b/poetry/core/lock/__init__.py @@ -0,0 +1,8 @@ +from poetry.core.lock.categories import LockCategory +from poetry.core.lock.locker import Locker + + +__all__ = [ + LockCategory.__name__, + Locker.__name__, +] diff --git a/poetry/core/lock/categories.py b/poetry/core/lock/categories.py new file mode 100644 index 000000000..c55d8a73c --- /dev/null +++ b/poetry/core/lock/categories.py @@ -0,0 +1,11 @@ +from enum import Enum + + +class LockCategory(Enum): + MAIN = "main" + DEV = "dev" + + def __eq__(self, other): + if not isinstance(other, Enum): + return self.value == other + super(LockCategory, self).__eq__(other) diff --git a/poetry/core/lock/locker.py b/poetry/core/lock/locker.py new file mode 100644 index 000000000..67e2fc26a --- /dev/null +++ b/poetry/core/lock/locker.py @@ -0,0 +1,397 @@ +import json +import logging +import os +import re + +from hashlib import sha256 +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Union + +from tomlkit import array +from tomlkit import document +from tomlkit import inline_table +from tomlkit import item +from tomlkit import table +from tomlkit.container import Container as TOMLContainer +from tomlkit.exceptions import TOMLKitError + +from poetry.core.lock.categories import LockCategory +from poetry.core.packages.package import Dependency +from poetry.core.packages.package import Package +from poetry.core.pyproject import PyProjectTOML +from poetry.core.semver import parse_constraint +from poetry.core.semver.version import Version +from poetry.core.toml import TOMLFile +from poetry.core.utils._compat import OrderedDict # noqa +from poetry.core.utils._compat import Path # noqa +from poetry.core.version.markers import parse_marker + + +logger = logging.getLogger(__name__) + + +Data = Union[Dict[str, Any], TOMLContainer] + + +class Locker(object): + + _VERSION = "1.1" + + _relevant_keys = ["dependencies", "dev-dependencies", "source", "extras"] + + def __init__(self, lock, local_config=None): # type: (Path, Optional[Data]) -> None + self._lock = TOMLFile(lock) + self._local_config = local_config + self._lock_data = None + self._content_hash = self._get_content_hash() + + @property + def lock(self): # type: () -> TOMLFile + return self._lock + + @property + def lock_data(self): + if self._lock_data is None: + self._lock_data = self._get_lock_data() + + return self._lock_data + + def is_locked(self): # type: () -> bool + """ + Checks whether the locker has been locked (lockfile found). + """ + if not self._lock.exists(): + return False + + return "package" in self.lock_data + + def is_fresh(self): # type: () -> bool + """ + Checks whether the lock file is still up to date with the current hash. + """ + lock = self._lock.read() + metadata = lock.get("metadata", {}) + + if "content-hash" in metadata: + return self._content_hash == lock["metadata"]["content-hash"] + + return False + + def get_packages( + self, names=None, categories=None + ): # type: (Optional[List[str]], Optional[List[LockCategory]]) -> List[Package] + """ + Get locked packages. Filters by categories if specified. + + :param names: Package names to filter on. + :param categories: Package categories to filter on. + """ + packages = [] + + if not self.is_locked(): + return packages + + locked_packages = [ + pkg + for pkg in self.lock_data["package"] + if (names is None or pkg["name"] in names) + and (categories is None or pkg["category"] in categories) + ] + + for info in locked_packages: + packages.append(self._load_package(info)) + + return packages + + def set_lock_data(self, root, packages): # type: (...) -> bool + files = table() + packages = self._lock_packages(packages) + # Retrieving hashes + for package in packages: + if package["name"] not in files: + files[package["name"]] = [] + + for f in package["files"]: + file_metadata = inline_table() + for k, v in sorted(f.items()): + file_metadata[k] = v + + files[package["name"]].append(file_metadata) # noqa + + if files[package["name"]]: + files[package["name"]] = item(files[package["name"]]).multiline(True) + + del package["files"] + + lock = document() + lock["package"] = packages + + if root.extras: + lock["extras"] = { + extra: [dep.pretty_name for dep in deps] + for extra, deps in sorted(root.extras.items()) + } + + lock["metadata"] = OrderedDict( + [ + ("lock-version", self._VERSION), + ("python-versions", root.python_versions), + ("content-hash", self._content_hash), + ("files", files), + ] + ) + + if not self.is_locked() or lock != self.lock_data: + self._write_lock_data(lock) + + return True + + return False + + def _write_lock_data(self, data): + self.lock.write(data) + + # Checking lock file data consistency + if data != self.lock.read(): + raise RuntimeError("Inconsistent lock file data.") + + self._lock_data = None + + def _get_content_hash(self): # type: () -> str + """ + Returns the sha256 hash of the sorted content of the pyproject file. + """ + if self._local_config is None: + return "" + + content = self._local_config + + relevant_content = {} + for key in self._relevant_keys: + relevant_content[key] = content.get(key) + + content_hash = sha256( + json.dumps(relevant_content, sort_keys=True).encode() + ).hexdigest() + + return content_hash + + def _get_lock_data(self): # type: () -> dict + if not self._lock.exists(): + raise RuntimeError("No lockfile found. Unable to read locked packages") + + try: + lock_data = self._lock.read() + except TOMLKitError as e: + raise RuntimeError("Unable to read the lock file ({}).".format(e)) + + lock_version = Version.parse(lock_data["metadata"].get("lock-version", "1.0")) + current_version = Version.parse(self._VERSION) + # We expect the locker to be able to read lock files + # from the same semantic versioning range + accepted_versions = parse_constraint( + "^{}".format(Version(current_version.major, 0)) + ) + lock_version_allowed = accepted_versions.allows(lock_version) + if lock_version_allowed and current_version < lock_version: + logger.warning( + "The lock file might not be compatible with the current version of Poetry.\n" + "Upgrade Poetry to ensure the lock file is read properly or, alternatively, " + "regenerate the lock file with the `poetry lock` command." + ) + elif not lock_version_allowed: + raise RuntimeError( + "The lock file is not compatible with the current version of Poetry.\n" + "Upgrade Poetry to be able to read the lock file or, alternatively, " + "regenerate the lock file with the `poetry lock` command." + ) + + return lock_data + + def _lock_packages(self, packages): # type: (List[Package]) -> List[Dict[str, Any]] + locked = [] + + for package in sorted(packages, key=lambda x: x.name): + spec = self._dump_package(package) + + locked.append(spec) + + return locked + + def _dump_package(self, package): # type: (Package) -> dict + dependencies = {} + for dependency in sorted(package.requires, key=lambda d: d.name): + if dependency.pretty_name not in dependencies: + dependencies[dependency.pretty_name] = [] + + constraint = inline_table() + constraint["version"] = str(dependency.pretty_constraint) + + if dependency.extras: + constraint["extras"] = sorted(dependency.extras) + + if dependency.is_optional(): + constraint["optional"] = True + + if not dependency.marker.is_any(): + constraint["markers"] = str(dependency.marker) + + dependencies[dependency.pretty_name].append(constraint) + + # All the constraints should have the same type, + # but we want to simplify them if it's possible + for dependency, constraints in tuple(dependencies.items()): + if all(len(constraint) == 1 for constraint in constraints): + dependencies[dependency] = [ + constraint["version"] for constraint in constraints + ] + + data = OrderedDict( + [ + ("name", package.pretty_name), + ("version", package.pretty_version), + ("description", package.description or ""), + ("category", package.category), + ("optional", package.optional), + ("python-versions", package.python_versions), + ("files", sorted(package.files, key=lambda x: x["file"])), + ] + ) + + if dependencies: + data["dependencies"] = table() + for k, constraints in dependencies.items(): + if len(constraints) == 1: + data["dependencies"][k] = constraints[0] + else: + data["dependencies"][k] = array().multiline(True) + for constraint in constraints: + data["dependencies"][k].append(constraint) + + if package.extras: + extras = {} + for name, deps in package.extras.items(): + extras[name] = [ + str(dep) if not dep.constraint.is_any() else dep.name + for dep in deps + ] + + data["extras"] = extras + + if package.source_url: + url = package.source_url + if package.source_type in ["file", "directory"]: + # The lock file should only store paths relative to the root project + url = Path( + os.path.relpath( + Path(url).as_posix(), self._lock.path.parent.as_posix() + ) + ).as_posix() + + data["source"] = OrderedDict() + + if package.source_type: + data["source"]["type"] = package.source_type + + data["source"]["url"] = url + + if package.source_reference: + data["source"]["reference"] = package.source_reference + + if package.source_resolved_reference: + data["source"]["resolved_reference"] = package.source_resolved_reference + + if package.source_type == "directory": + data["develop"] = package.develop + + return data + + def _load_package(self, info): # type: (Data) -> Package + from poetry.core.factory import Factory + + lock_metadata = self.lock_data["metadata"] + source = info.get("source", {}) + source_type = source.get("type") + url = source.get("url") + if source_type in ["directory", "file"]: + url = self._lock.path.parent.joinpath(url).resolve().as_posix() + + package = Package( + info["name"], + info["version"], + info["version"], + source_type=source_type, + source_url=url, + source_reference=source.get("reference"), + source_resolved_reference=source.get("resolved_reference"), + ) + package.description = info.get("description", "") + package.category = info["category"] + package.optional = info["optional"] + if "hashes" in lock_metadata: + # Old lock so we create dummy files from the hashes + package.files = [ + {"name": h, "hash": h} for h in lock_metadata["hashes"][info["name"]] + ] + else: + package.files = lock_metadata["files"][info["name"]] + + package.python_versions = info["python-versions"] + extras = info.get("extras", {}) + if extras: + for name, deps in extras.items(): + package.extras[name] = [] + + for dep in deps: + m = re.match(r"^(.+?)(?:\s+\((.+)\))?$", dep) + dep_name = m.group(1) + constraint = m.group(2) or "*" + + package.extras[name].append(Dependency(dep_name, constraint)) + + if "marker" in info: + package.marker = parse_marker(info["marker"]) + else: + # Compatibility for old locks + if "requirements" in info: + dep = Dependency("foo", "0.0.0") + for name, value in info["requirements"].items(): + if name == "python": + dep.python_versions = value + elif name == "platform": + dep.platform = value + + split_dep = dep.to_pep_508(False).split(";") + if len(split_dep) > 1: + package.marker = parse_marker(split_dep[1].strip()) + + for dep_name, constraint in info.get("dependencies", {}).items(): + if isinstance(constraint, list): + for c in constraint: + package.add_dependency( + Factory.create_dependency( + dep_name, c, root_dir=self._lock.path.parent + ) + ) + + continue + + package.add_dependency( + Factory.create_dependency( + dep_name, constraint, root_dir=self._lock.path.parent + ) + ) + + if "develop" in info: + package.develop = info["develop"] + + return package + + @classmethod + def load(cls, lock, pyproject_file=None): # type: (Path, Optional[Path]) -> Locker + if pyproject_file and pyproject_file.exists(): + return cls(lock, PyProjectTOML(pyproject_file).poetry_config) + return cls(lock) diff --git a/poetry/core/poetry.py b/poetry/core/poetry.py index af04a0dad..f8dd8039c 100644 --- a/poetry/core/poetry.py +++ b/poetry/core/poetry.py @@ -3,23 +3,26 @@ from typing import TYPE_CHECKING from typing import Any +from typing import Optional from poetry.core.pyproject import PyProjectTOML from poetry.core.utils._compat import Path # noqa if TYPE_CHECKING: + from poetry.core.lock.locker import Locker from poetry.core.packages import ProjectPackage # noqa from poetry.core.pyproject.toml import PyProjectTOMLFile # noqa class Poetry(object): def __init__( - self, file, local_config, package, - ): # type: (Path, dict, "ProjectPackage") -> None + self, file, local_config, package, locker=None + ): # type: (Path, dict, "ProjectPackage", Optional["Locker"]) -> None self._pyproject = PyProjectTOML(file) self._package = package self._local_config = local_config + self._locker = locker @property def pyproject(self): # type: () -> PyProjectTOML @@ -37,5 +40,15 @@ def package(self): # type: () -> "ProjectPackage" def local_config(self): # type: () -> dict return self._local_config + @property + def locker(self): # type: () -> Optional["Locker"] + if self._locker is None: + from poetry.core.lock.locker import Locker + + self._locker = Locker( + self.pyproject.file / "poetry.lock", self.local_config + ) + return self._locker + def get_project_config(self, config, default=None): # type: (str, Any) -> Any return self._local_config.get("config", {}).get(config, default) diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py new file mode 100644 index 000000000..884484a6a --- /dev/null +++ b/tests/packages/test_locker.py @@ -0,0 +1,398 @@ +import logging +import tempfile + +import pytest +import tomlkit + +from poetry.core.factory import Factory +from poetry.core.lock import Locker +from poetry.core.packages.package import Package +from poetry.core.packages.project_package import ProjectPackage +from poetry.core.semver.version import Version +from poetry.core.utils._compat import Path # noqa + + +@pytest.fixture +def locker(): + with tempfile.NamedTemporaryFile() as f: + f.close() + locker = Locker(Path(f.name), {}) + + return locker + + +@pytest.fixture +def root(): + return ProjectPackage("root", "1.2.3") + + +def test_lock_file_data_is_ordered(locker, root): + package_a = Package("A", "1.0.0") + package_a.add_dependency(Factory.create_dependency("B", "^1.0")) + package_a.files = [{"file": "foo", "hash": "456"}, {"file": "bar", "hash": "123"}] + package_git = Package( + "git-package", + "1.2.3", + source_type="git", + source_url="https://github.com/python-poetry/poetry.git", + source_reference="develop", + source_resolved_reference="123456", + ) + packages = [package_a, Package("B", "1.2"), package_git] + + locker.set_lock_data(root, packages) + + with locker.lock.open(encoding="utf-8") as f: + content = f.read() + + expected = """[[package]] +name = "A" +version = "1.0.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +B = "^1.0" + +[[package]] +name = "B" +version = "1.2" +description = "" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "git-package" +version = "1.2.3" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.source] +type = "git" +url = "https://github.com/python-poetry/poetry.git" +reference = "develop" +resolved_reference = "123456" + +[metadata] +lock-version = "1.1" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" + +[metadata.files] +A = [ + {file = "bar", hash = "123"}, + {file = "foo", hash = "456"}, +] +B = [] +git-package = [] +""" + + assert expected == content + + +def test_locker_properly_loads_extras(locker): + content = """\ +[[package]] +name = "cachecontrol" +version = "0.12.5" +description = "httplib2 caching for requests" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +msgpack = "*" +requests = "*" + +[package.dependencies.lockfile] +optional = true +version = ">=0.9" + +[package.extras] +filecache = ["lockfile (>=0.9)"] +redis = ["redis (>=2.10.5)"] + +[metadata] +lock-version = "1.1" +python-versions = "~2.7 || ^3.4" +content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77" + +[metadata.files] +cachecontrol = [] +""" + + locker.lock.write(tomlkit.parse(content)) + + packages = locker.get_packages() + + assert 1 == len(packages) + + package = packages[0] + assert 3 == len(package.requires) + assert 2 == len(package.extras) + + lockfile_dep = package.extras["filecache"][0] + assert lockfile_dep.name == "lockfile" + + +def test_lock_packages_with_null_description(locker, root): + package_a = Package("A", "1.0.0") + package_a.description = None + + locker.set_lock_data(root, [package_a]) + + with locker.lock.open(encoding="utf-8") as f: + content = f.read() + + expected = """[[package]] +name = "A" +version = "1.0.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +lock-version = "1.1" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" + +[metadata.files] +A = [] +""" + + assert expected == content + + +def test_lock_file_should_not_have_mixed_types(locker, root): + package_a = Package("A", "1.0.0") + package_a.add_dependency(Factory.create_dependency("B", "^1.0.0")) + package_a.add_dependency( + Factory.create_dependency("B", {"version": ">=1.0.0", "optional": True}) + ) + package_a.requires[-1].activate() + package_a.extras["foo"] = [Factory.create_dependency("B", ">=1.0.0")] + + locker.set_lock_data(root, [package_a]) + + expected = """[[package]] +name = "A" +version = "1.0.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +B = [ + {version = "^1.0.0"}, + {version = ">=1.0.0", optional = true}, +] + +[package.extras] +foo = ["B (>=1.0.0)"] + +[metadata] +lock-version = "1.1" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" + +[metadata.files] +A = [] +""" + + with locker.lock.open(encoding="utf-8") as f: + content = f.read() + + assert expected == content + + +def test_reading_lock_file_should_raise_an_error_on_invalid_data(locker): + content = u"""[[package]] +name = "A" +version = "1.0.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +foo = ["bar"] + +[package.extras] +foo = ["bar"] + +[metadata] +lock-version = "1.1" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" + +[metadata.files] +A = [] +""" + with locker.lock.open("w", encoding="utf-8") as f: + f.write(content) + + with pytest.raises(RuntimeError) as e: + _ = locker.lock_data + + assert "Unable to read the lock file" in str(e.value) + + +def test_locking_legacy_repository_package_should_include_source_section(root, locker): + package_a = Package( + "A", + "1.0.0", + source_type="legacy", + source_url="https://foo.bar", + source_reference="legacy", + ) + packages = [package_a] + + locker.set_lock_data(root, packages) + + with locker.lock.open(encoding="utf-8") as f: + content = f.read() + + expected = """[[package]] +name = "A" +version = "1.0.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.source] +type = "legacy" +url = "https://foo.bar" +reference = "legacy" + +[metadata] +lock-version = "1.1" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" + +[metadata.files] +A = [] +""" + + assert expected == content + + +def test_locker_should_emit_warnings_if_lock_version_is_newer_but_allowed( + locker, caplog +): + content = """\ +[metadata] +lock-version = "{version}" +python-versions = "~2.7 || ^3.4" +content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77" + +[metadata.files] +""".format( + version=".".join(Version.parse(Locker._VERSION).next_minor.text.split(".")[:2]) + ) + caplog.set_level(logging.WARNING, logger="poetry.packages.locker") + + locker.lock.write(tomlkit.parse(content)) + + _ = locker.lock_data + + assert 1 == len(caplog.records) + + record = caplog.records[0] + assert "WARNING" == record.levelname + + expected = """\ +The lock file might not be compatible with the current version of Poetry. +Upgrade Poetry to ensure the lock file is read properly or, alternatively, \ +regenerate the lock file with the `poetry lock` command.\ +""" + assert expected == record.message + + +def test_locker_should_raise_an_error_if_lock_version_is_newer_and_not_allowed( + locker, caplog +): + content = """\ +[metadata] +lock-version = "2.0" +python-versions = "~2.7 || ^3.4" +content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77" + +[metadata.files] +""" + caplog.set_level(logging.WARNING, logger="poetry.packages.locker") + + locker.lock.write(tomlkit.parse(content)) + + with pytest.raises(RuntimeError, match="^The lock file is not compatible"): + _ = locker.lock_data + + +def test_extras_dependencies_are_ordered(locker, root): + package_a = Package("A", "1.0.0") + package_a.add_dependency( + Factory.create_dependency( + "B", {"version": "^1.0.0", "optional": True, "extras": ["c", "a", "b"]} + ) + ) + package_a.requires[-1].activate() + + locker.set_lock_data(root, [package_a]) + + expected = """[[package]] +name = "A" +version = "1.0.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +B = {version = "^1.0.0", extras = ["a", "b", "c"], optional = true} + +[metadata] +lock-version = "1.1" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" + +[metadata.files] +A = [] +""" + + with locker.lock.open(encoding="utf-8") as f: + content = f.read() + + assert expected == content + + +def test_locker_should_neither_emit_warnings_nor_raise_error_for_lower_compatible_versions( + locker, caplog +): + current_version = Version.parse(Locker._VERSION) + older_version = ".".join( + [str(current_version.major), str(current_version.minor - 1)] + ) + content = """\ +[metadata] +lock-version = "{version}" +python-versions = "~2.7 || ^3.4" +content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77" + +[metadata.files] +""".format( + version=older_version + ) + caplog.set_level(logging.WARNING, logger="poetry.packages.locker") + + locker.lock.write(tomlkit.parse(content)) + + _ = locker.lock_data + + assert 0 == len(caplog.records)