From 550f08a455eeb9e3dc6872c4d3645c0ee5e206c1 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Fri, 5 Jan 2024 11:57:20 +0100 Subject: [PATCH 1/7] Install importlib_(metadata|resources) on older Pythons --- pyproject.toml | 2 ++ requirements/base.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c9d481806b..de26325251 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ classifiers = [ dynamic = ["version"] dependencies = [ "pydantic >= 2.0", + "importlib_metadata; python_version < '3.8'", + "importlib_resources; python_version < '3.9'", ] diff --git a/requirements/base.txt b/requirements/base.txt index 87041f9f77..6a949e456d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -37,4 +37,6 @@ libsass==0.15.1 napalm==3.4.1 backports.zoneinfo ; python_version < '3.9' +importlib_metadata ; python_version < '3.8' +importlib_resources ; python_version < '3.9' git+https://github.com/Uninett/drf-oidc-auth@v4.0#egg=drf-oidc-auth From 833f6b45a86f70348888f72ea2bb092789397ed3 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Fri, 5 Jan 2024 10:50:59 +0100 Subject: [PATCH 2/7] Replace usage of pkg_resources.resource_filename --- python/nav/statemon/checker/RadiusChecker.py | 2 +- python/nav/util.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/nav/statemon/checker/RadiusChecker.py b/python/nav/statemon/checker/RadiusChecker.py index eeed2c322e..51b69adef5 100644 --- a/python/nav/statemon/checker/RadiusChecker.py +++ b/python/nav/statemon/checker/RadiusChecker.py @@ -14,7 +14,7 @@ # License along with NAV. If not, see . # """RADIUS service checker""" -from pkg_resources import resource_filename +from nav.util import resource_filename # Python-radius specific modules. pyrad found at # http://www.wiggy.net/code/pyrad/ by Wichert Akkermann diff --git a/python/nav/util.py b/python/nav/util.py index c7b609d309..cc01dcb86b 100644 --- a/python/nav/util.py +++ b/python/nav/util.py @@ -30,6 +30,11 @@ import IPy +try: + from importlib.resources import files, as_file +except ImportError: # Python 3.7! + from importlib_resources import files, as_file + def gradient(start, stop, steps): """Create and return a sequence of steps representing an integer @@ -509,3 +514,9 @@ def _range_to_str(x, y): return str(x) else: return "{}-{}".format(x, y) + + +def resource_filename(package, filename): + ref = files(package) / filename + with as_file(ref) as path: + return str(path) From 945ba7361d8260c9cb8adf7820ec1dfe01ad0dae Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Fri, 5 Jan 2024 11:03:08 +0100 Subject: [PATCH 3/7] Replace usage of pkg_resources.resource_string --- python/nav/config.py | 9 ++++++++- python/nav/pgsync.py | 11 +++++++++-- python/nav/util.py | 5 +++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/python/nav/config.py b/python/nav/config.py index 9a500a93b9..55615af35b 100644 --- a/python/nav/config.py +++ b/python/nav/config.py @@ -30,8 +30,15 @@ import pkg_resources from nav.errors import GeneralException +from nav.util import files, resource_bytes from . import buildconf +try: + from importlib.resources import files +except ImportError: # Python 3.7! + from importlib_resources import files + + _logger = logging.getLogger(__name__) # Potential locations to find configuration files @@ -272,7 +279,7 @@ def _install_single_config_resource_(source, target, overwrite=False): if not overwrite and os.path.exists(target_file): return False - content = pkg_resources.resource_string('nav', resource_path) + content = resource_bytes('nav', resource_path) with open(target_file, 'wb') as handle: handle.write(content) return target_file diff --git a/python/nav/pgsync.py b/python/nav/pgsync.py index 6b8386b969..72e86a02d4 100755 --- a/python/nav/pgsync.py +++ b/python/nav/pgsync.py @@ -26,12 +26,19 @@ from textwrap import wrap from errno import ENOENT, EACCES import psycopg2 -from pkg_resources import resource_listdir, resource_string +from pkg_resources import resource_listdir from nav.db import ConnectionParameters from nav.colors import colorize, print_color from nav.colors import COLOR_CYAN, COLOR_YELLOW, COLOR_RED, COLOR_GREEN +try: + from importlib.resources import files +except ImportError: # Python 3.7! + from importlib_resources import files + +from nav.util import files, resource_bytes + def main(): """Main program""" @@ -546,7 +553,7 @@ def execute_sql_file(self, filename): print_color("OK", COLOR_GREEN) def _read_sql_file(self, filename): - return resource_string(self.resource_module, filename) + return resource_bytes(self.resource_module, filename) class ChangeScriptFinder(list): diff --git a/python/nav/util.py b/python/nav/util.py index cc01dcb86b..c10f7d28f4 100644 --- a/python/nav/util.py +++ b/python/nav/util.py @@ -520,3 +520,8 @@ def resource_filename(package, filename): ref = files(package) / filename with as_file(ref) as path: return str(path) + + +def resource_bytes(package, filename): + ref = files(package) / filename + return ref.read_bytes() From aa66c4527a4515d5b6f99c346641fa98716d6f92 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Fri, 5 Jan 2024 10:56:21 +0100 Subject: [PATCH 4/7] Replace usage of pkg_resources.resource_(listdir|isdir) --- python/nav/config.py | 22 +++++++++------------- python/nav/pgsync.py | 21 ++++++++------------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/python/nav/config.py b/python/nav/config.py index 55615af35b..b55dd402ca 100644 --- a/python/nav/config.py +++ b/python/nav/config.py @@ -27,18 +27,12 @@ import pwd import stat import configparser -import pkg_resources +from pathlib import Path from nav.errors import GeneralException from nav.util import files, resource_bytes from . import buildconf -try: - from importlib.resources import files -except ImportError: # Python 3.7! - from importlib_resources import files - - _logger = logging.getLogger(__name__) # Potential locations to find configuration files @@ -254,13 +248,15 @@ def _config_resource_walk(source=''): from available nav package resources. All paths returned will be relative to the etc top directory. """ - current_path = os.path.join('etc', source) - for name in pkg_resources.resource_listdir('nav', current_path): - full_name = os.path.join(current_path, name) - relative_name = os.path.join(source, name) - if pkg_resources.resource_isdir('nav', full_name): + source = Path(source) + current_path = Path('etc') / source + for path in files('nav').joinpath(current_path).iterdir(): + name = path.name + full_name = current_path / name + relative_name = str(source / name) + if files('nav').joinpath(full_name).is_dir(): for path in _config_resource_walk(source=relative_name): - yield path + yield str(path) else: yield relative_name diff --git a/python/nav/pgsync.py b/python/nav/pgsync.py index 72e86a02d4..7ca81143cc 100755 --- a/python/nav/pgsync.py +++ b/python/nav/pgsync.py @@ -25,18 +25,12 @@ import subprocess from textwrap import wrap from errno import ENOENT, EACCES +from pathlib import Path import psycopg2 -from pkg_resources import resource_listdir from nav.db import ConnectionParameters from nav.colors import colorize, print_color from nav.colors import COLOR_CYAN, COLOR_YELLOW, COLOR_RED, COLOR_GREEN - -try: - from importlib.resources import files -except ImportError: # Python 3.7! - from importlib_resources import files - from nav.util import files, resource_bytes @@ -569,12 +563,13 @@ def __init__(self, resource_module): self._find_change_scripts() def _find_change_scripts(self): - changes_dir = 'sql/changes' - scripts = [ - os.path.join(changes_dir, f) - for f in resource_listdir(self.resource_module, changes_dir) - if self.script_pattern.match(f) - ] + changes_dir = Path('sql/changes') + scripts = [] + sql_path = files(self.resource_module).joinpath(changes_dir) + for path in sql_path.iterdir(): + filename = path.name + if self.script_pattern.match(str(filename)): + scripts.append(str(changes_dir / filename)) self[:] = scripts def get_missing_changes(self, versions): From 8e584e52484a416ee31c006a82caafdb31f8e273 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Fri, 5 Jan 2024 12:33:45 +0100 Subject: [PATCH 5/7] Replace usage of pkg_resources.get_distribution --- python/nav/buildconf.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/nav/buildconf.py b/python/nav/buildconf.py index 16650c650c..c39aed4088 100644 --- a/python/nav/buildconf.py +++ b/python/nav/buildconf.py @@ -2,7 +2,12 @@ # pylint: disable=invalid-name import os import sysconfig -import pkg_resources + +try: + from importlib import metadata as _impmeta +except ImportError: + import importlib_metadata as _impmeta + datadir = os.path.join(sysconfig.get_config_var('datarootdir'), 'nav') localstatedir = os.path.join(datadir, 'var') @@ -10,9 +15,10 @@ djangotmpldir = os.path.join(datadir, "templates") docdir = os.path.join(datadir, "doc") + try: - VERSION = pkg_resources.get_distribution("nav").version -except pkg_resources.DistributionNotFound: + VERSION = _impmeta.version("nav") +except _impmeta.PackageNotFoundError: # If we're not installed, try to get the current version from Git tags import setuptools_scm From ee1c136bcf762bc6e8d1b50c615cb9c950da212d Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 26 Feb 2024 14:08:54 +0100 Subject: [PATCH 6/7] Rename files to resource_files --- python/nav/config.py | 6 +++--- python/nav/pgsync.py | 4 ++-- python/nav/util.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/nav/config.py b/python/nav/config.py index b55dd402ca..4fe1f00d5e 100644 --- a/python/nav/config.py +++ b/python/nav/config.py @@ -30,7 +30,7 @@ from pathlib import Path from nav.errors import GeneralException -from nav.util import files, resource_bytes +from nav.util import resource_files, resource_bytes from . import buildconf _logger = logging.getLogger(__name__) @@ -250,11 +250,11 @@ def _config_resource_walk(source=''): """ source = Path(source) current_path = Path('etc') / source - for path in files('nav').joinpath(current_path).iterdir(): + for path in resource_files('nav').joinpath(current_path).iterdir(): name = path.name full_name = current_path / name relative_name = str(source / name) - if files('nav').joinpath(full_name).is_dir(): + if resource_files('nav').joinpath(full_name).is_dir(): for path in _config_resource_walk(source=relative_name): yield str(path) else: diff --git a/python/nav/pgsync.py b/python/nav/pgsync.py index 7ca81143cc..54d0a8c50a 100755 --- a/python/nav/pgsync.py +++ b/python/nav/pgsync.py @@ -31,7 +31,7 @@ from nav.db import ConnectionParameters from nav.colors import colorize, print_color from nav.colors import COLOR_CYAN, COLOR_YELLOW, COLOR_RED, COLOR_GREEN -from nav.util import files, resource_bytes +from nav.util import resource_files, resource_bytes def main(): @@ -565,7 +565,7 @@ def __init__(self, resource_module): def _find_change_scripts(self): changes_dir = Path('sql/changes') scripts = [] - sql_path = files(self.resource_module).joinpath(changes_dir) + sql_path = resource_files(self.resource_module).joinpath(changes_dir) for path in sql_path.iterdir(): filename = path.name if self.script_pattern.match(str(filename)): diff --git a/python/nav/util.py b/python/nav/util.py index c10f7d28f4..5064e08217 100644 --- a/python/nav/util.py +++ b/python/nav/util.py @@ -31,9 +31,9 @@ import IPy try: - from importlib.resources import files, as_file + from importlib.resources import as_file, files as resource_files except ImportError: # Python 3.7! - from importlib_resources import files, as_file + from importlib_resources import as_file, files as resource_files def gradient(start, stop, steps): @@ -517,11 +517,11 @@ def _range_to_str(x, y): def resource_filename(package, filename): - ref = files(package) / filename + ref = resource_files(package) / filename with as_file(ref) as path: return str(path) def resource_bytes(package, filename): - ref = files(package) / filename + ref = resource_files(package) / filename return ref.read_bytes() From 9a871c951c18fcb63a9cf2926dec9bddeedb3532 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 26 Feb 2024 14:09:24 +0100 Subject: [PATCH 7/7] Add docstrings to resource-functions --- python/nav/util.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/nav/util.py b/python/nav/util.py index 5064e08217..edb8d9c93f 100644 --- a/python/nav/util.py +++ b/python/nav/util.py @@ -517,11 +517,21 @@ def _range_to_str(x, y): def resource_filename(package, filename): + """Return the path of the filename as it is inside the package + + package: either a dotted path to a module or a module object + filename: str or pathlib.Path + """ ref = resource_files(package) / filename with as_file(ref) as path: return str(path) def resource_bytes(package, filename): + """Read and return a bytes-object of the filename found in the package + + package: either a dotted path to a module or a module object + filename: str or pathlib.Path + """ ref = resource_files(package) / filename return ref.read_bytes()