diff --git a/.editorconfig b/.editorconfig index 2f745f2..45a96d6 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,3 +8,6 @@ end_of_line = lf charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true + +[*.yml] +indent_size = 2 diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml new file mode 100644 index 0000000..c6544d8 --- /dev/null +++ b/.github/workflows/ci_tests.yml @@ -0,0 +1,23 @@ +name: CI tests + +on: + pull_request: + workflow_dispatch: + +jobs: + build_package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build package + run: python -m build diff --git a/.github/workflows/publish_release_to_pypi.yml b/.github/workflows/publish_release_to_pypi.yml new file mode 100644 index 0000000..b75cc1e --- /dev/null +++ b/.github/workflows/publish_release_to_pypi.yml @@ -0,0 +1,29 @@ +name: Publish release to PyPI + +on: + release: + types: + - published + +jobs: + publish_package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build package + run: python -m build + + - name: Publish package + uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index 3523baf..202ccb8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ +dist/ *.egg-info/ .venv/ venv/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad04e8..ac35926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,22 +5,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/brooklyn-data/dbtenv/compare/v1.1.1...HEAD) + +## [Unreleased](https://github.com/brooklyn-data/dbtenv/compare/v1.2.0...HEAD) + +### Added + +### Changed + +### Fixed + + +## [1.2.0](https://github.com/brooklyn-data/dbtenv/compare/v1.1.1...v1.2.0) - 2021-11-29 ### Added +- New `--quiet` argument to not output any nonessential information as dbtenv runs. +- Allow location of dbt version-specific Python virtual environments to be configured with `DBTENV_VENVS_DIRECTORY` and `DBTENV_VENVS_PREFIX` environment variables. +- Publish dbtenv package to PyPI. ### Changed +- If no specific dbt version has been selected then default to using the max installed version (if any) or the max installable version (preferring stable versions). +- If no compatible dbt version can be found for a dbt project and its installed packages then ignore dbt version requirements from installed packages in case they're simply out of date. +- When installing with pip, upgrade pip to avoid problems with packages that might require newer pip features. +- When installing with Homebrew, automatically add the dbt Homebrew tap if necessary. +- Switch from distutils to setuptools. + ## [1.1.1](https://github.com/brooklyn-data/dbtenv/compare/v1.1.0...v1.1.1) - 2021-07-15 ### Fixed - Fix error when `~/.dbt/versions` directory doesn't exist yet. + ## [1.1.0](https://github.com/brooklyn-data/dbtenv/compare/v1.0.0...v1.1.0) - 2021-07-14 ### Added - Support installation of dbt versions >= 0.20.0 in a Python 3.9 environment. + ## [1.0.0](https://github.com/brooklyn-data/dbtenv/releases/tag/v1.0.0) - 2021-04-16 ### Added diff --git a/README.md b/README.md index 06859b2..fc15129 100644 --- a/README.md +++ b/README.md @@ -2,22 +2,11 @@ dbtenv lets you easily install and run multiple versions of [dbt](https://docs.getdbt.com/docs/introduction) using [pip](https://pip.pypa.io/) with [Python virtual environments](https://docs.python.org/3/library/venv.html), or optionally using [Homebrew](https://brew.sh/) on Mac or Linux. -### Table of contents -- **[Installation](#installation)** -- **[How it works](#how-it-works)** - - [Using pip and/or Homebrew](#using-pip-andor-homebrew) - - [Installing dbt versions](#installing-dbt-versions) - - [Switching between dbt versions](#switching-between-dbt-versions) - - [Running dbt versions](#running-dbt-versions) - - [Running dbt with dbtenv more seamlessly](#running-dbt-with-dbtenv-more-seamlessly) - - [Uninstalling dbt versions](#uninstalling-dbt-versions) -- **[Development](#development)** - - [Development setup](#development-setup) - ## Installation -Run `pip3 install git+https://github.com/brooklyn-data/dbtenv#egg=dbtenv`. +1. Install [pipx](https://pypa.github.io/pipx/) if you haven't already. +2. Run `pipx install dbtenv`. ## How it works @@ -35,6 +24,7 @@ You can run `dbtenv versions` to list the versions of dbt available to install, If you don't want to have to run `dbtenv install ` manually, you can set a `DBTENV_AUTO_INSTALL=true` environment variable so that as you run commands like `dbtenv version` or `dbtenv execute` any dbt version specified that isn't already installed will be installed automatically. Some tips when dbtenv is using pip: +- You can customize where the dbt version-specific Python virtual environments are created by setting `DBTENV_VENVS_DIRECTORY` and `DBTENV_VENVS_PREFIX` environment variables. - You can have dbtenv only install Python packages that were actually available on the date the dbt version was released by setting a `DBTENV_SIMULATE_RELEASE_DATE=true` environment variable, or specifying `--simulate-release-date` when running `dbtenv install`. This can help if newer versions of dbt's dependencies are causing installation problems. - By default dbtenv uses whichever Python version it was installed with to install dbt, but that can be changed by setting a `DBTENV_PYTHON` environment variable to the path of a different Python executable, or specifying `--python ` when running `dbtenv install`. @@ -49,6 +39,8 @@ dbtenv determines which dbt version to use by trying to read it from the followi - If the dbt version requirements specify a range of versions rather than an exact version, then dbtenv will try to read a preferred dbt version from the sources below and will use that version if it's compatible with the requirements. 5. The first `.dbt_version` file found by searching the dbt project's parent directories. 6. The `~/.dbt/version` file. +7. The max installed dbt version (preferring stable versions). +8. The max installable dbt version (preferring stable versions). You can: - Run `dbtenv version` to show which dbt version dbtenv determines dynamically based on the current environment. @@ -81,7 +73,7 @@ Some examples: Note that after adding such a `dbt` alias/function to your shell profile you'll need to reload the profile to activate it (e.g. by running `. ~/.bash_profile` in bash, or `. $PROFILE` in PowerShell). ### Uninstalling dbt versions -You can run `dbtenv versions --installed` to list the versions of dbt that dbtenv has installed under `~/.dbt/versions` and/or with Homebrew, and then run `dbtenv uninstall ` to uninstall a version. +You can run `dbtenv versions --installed` to list the versions of dbt that dbtenv has installed in Python virtual environments and/or with Homebrew, and then run `dbtenv uninstall ` to uninstall a version. ## Development diff --git a/dbtenv/__init__.py b/dbtenv/__init__.py index f37e4fa..49c52aa 100644 --- a/dbtenv/__init__.py +++ b/dbtenv/__init__.py @@ -14,9 +14,10 @@ from typing import Any, List, Optional -__version__ = '1.1.1' +__version__ = '1.2.0' -VENVS_DIRECTORY = os.path.normpath('~/.dbt/versions') +DEFAULT_VENVS_DIRECTORY = os.path.normpath('~/.dbt/versions') +DEFAULT_VENVS_PREFIX = '' GLOBAL_VERSION_FILE = os.path.normpath('~/.dbt/version') LOCAL_VERSION_FILE = '.dbt_version' @@ -26,7 +27,10 @@ DEBUG_VAR = 'DBTENV_DEBUG' DEFAULT_INSTALLER_VAR = 'DBTENV_DEFAULT_INSTALLER' PYTHON_VAR = 'DBTENV_PYTHON' +QUIET_VAR = 'DBTENV_QUIET' SIMULATE_RELEASE_DATE_VAR = 'DBTENV_SIMULATE_RELEASE_DATE' +VENVS_DIRECTORY_VAR = 'DBTENV_VENVS_DIRECTORY' +VENVS_PREFIX_VAR = 'DBTENV_VENVS_PREFIX' def string_is_true(value: str) -> bool: @@ -71,9 +75,13 @@ def __str__(self) -> str: class Version(distutils.version.LooseVersion): - def __init__(self, version: str, source: Optional[str] = None) -> None: + def __init__(self, version: str, source: Optional[str] = None, source_description: Optional[str] = None) -> None: self.pypi_version = self.homebrew_version = self.raw_version = version.strip() self.source = source + if source and not source_description: + self.source_description = f"set by {source}" + else: + self.source_description = source_description version_match = re.match(r'(?P\d+\.\d+\.\d+)(-?(?P[a-z].*))?', self.raw_version) self.is_semantic = version_match is not None @@ -138,7 +146,9 @@ def __init__(self) -> None: self.project_file = self.find_file_along_working_path('dbt_project.yml') self.project_directory = os.path.dirname(self.project_file) if self.project_file else None - self.venvs_directory = os.path.expanduser(VENVS_DIRECTORY) + self.venvs_directory = os.path.expanduser(self.env_vars.get(VENVS_DIRECTORY_VAR) or DEFAULT_VENVS_DIRECTORY) + self.venvs_prefix = self.env_vars.get(VENVS_PREFIX_VAR) or DEFAULT_VENVS_PREFIX + self.global_version_file = os.path.expanduser(GLOBAL_VERSION_FILE) self.homebrew_installed = False @@ -166,7 +176,32 @@ def debug(self) -> bool: @debug.setter def debug(self, value: bool) -> None: self._debug = value - LOGGER.setLevel(logging.DEBUG if self._debug else logging.INFO) + self.update_logging_level() + + _quiet: Optional[bool] = None + + @property + def quiet(self) -> bool: + if self._quiet is None: + if QUIET_VAR in self.env_vars: + self._quiet = string_is_true(self.env_vars[QUIET_VAR]) + else: + self._quiet = False + + return self._quiet + + @quiet.setter + def quiet(self, value: bool) -> None: + self._quiet = value + self.update_logging_level() + + def update_logging_level(self) -> None: + if self.debug: + LOGGER.setLevel(logging.DEBUG) + elif self.quiet: + LOGGER.setLevel(logging.ERROR) + else: + LOGGER.setLevel(logging.INFO) _default_installer: Optional[Installer] = None diff --git a/dbtenv/execute.py b/dbtenv/execute.py index 8f41d8f..ff4aaf5 100644 --- a/dbtenv/execute.py +++ b/dbtenv/execute.py @@ -57,7 +57,7 @@ def execute(self, args: Args) -> None: version = args.dbt_version else: version = dbtenv.version.get_version(self.env) - logger.info(f"Using dbt {version} (set by {version.source}).") + logger.info(f"Using dbt {version} ({version.source_description}).") dbt = dbtenv.which.try_get_dbt(self.env, version) if not dbt: diff --git a/dbtenv/homebrew.py b/dbtenv/homebrew.py index 6044101..84dd072 100644 --- a/dbtenv/homebrew.py +++ b/dbtenv/homebrew.py @@ -27,7 +27,20 @@ def get_dbt_version_keg_directory(env: Environment, version: Version) -> str: return os.path.join(env.homebrew_prefix_directory, 'opt', get_dbt_version_formula(version)) +def ensure_homebrew_dbt_tap() -> None: + # While it would be simplest to just always run `brew tap dbt-labs/dbt`, we need to first check for + # an existing dbt tap because the "dbt-labs" GitHub organization used to be named "fishtown-analytics" + # so people could have a "fishtown-analytics/dbt" tap, and having multiple dbt taps causes errors. + tap_list_result = subprocess.run(['brew', 'tap'], stdout=subprocess.PIPE, text=True) + if not re.search(r'\b(fishtown-analytics|dbt-labs)/dbt\b', tap_list_result.stdout): + logger.info('Adding the dbt Homebrew tap.') + tap_dbt_result = subprocess.run(['brew', 'tap', 'dbt-labs/dbt']) + if tap_dbt_result.returncode != 0: + raise DbtenvError("Failed to add the dbt Homebrew tap.") + + def get_homebrew_dbt_versions() -> List[Version]: + ensure_homebrew_dbt_tap() brew_args = ['info', '--json', 'dbt'] logger.debug(f"Running `brew` with arguments {brew_args}.") brew_result = subprocess.run(['brew', *brew_args], stdout=subprocess.PIPE) @@ -59,6 +72,8 @@ def install(self, force: bool = False) -> None: self.uninstall(force=True) else: raise DbtenvError(f"dbt {self.version.homebrew_version} is already installed with Homebrew.") + else: + ensure_homebrew_dbt_tap() logger.info(f"Installing dbt {self.version.homebrew_version} with Homebrew.") brew_args = ['install', get_dbt_version_formula(self.version)] diff --git a/dbtenv/install.py b/dbtenv/install.py index 5b5daae..69da726 100644 --- a/dbtenv/install.py +++ b/dbtenv/install.py @@ -70,7 +70,7 @@ def execute(self, args: Args) -> None: version = args.dbt_version else: version = dbtenv.version.get_version(self.env) - logger.info(f"Using dbt {version} (set by {version.source}).") + logger.info(f"Using dbt {version} ({version.source_description}).") if self.env.primary_installer == Installer.PIP: pip_dbt = dbtenv.pip.PipDbt(self.env, version) diff --git a/dbtenv/main.py b/dbtenv/main.py index bbf82a2..429db25 100644 --- a/dbtenv/main.py +++ b/dbtenv/main.py @@ -30,7 +30,8 @@ def build_root_args_parser(env: Environment) -> argparse.ArgumentParser: description=f""" Lets you easily install and run multiple versions of dbt using pip with Python virtual environments, or optionally using Homebrew on Mac or Linux. - Any dbt version-specific Python virtual environments are created under `{dbtenv.VENVS_DIRECTORY}`. + Any dbt version-specific Python virtual environments are created under `{dbtenv.DEFAULT_VENVS_DIRECTORY}` by default, + but that can be configured using {dbtenv.VENVS_DIRECTORY_VAR} and {dbtenv.VENVS_PREFIX_VAR} environment variables. The dbt version to use can be configured in your shell using a {dbtenv.DBT_VERSION_VAR} environment variable, in dbt projects using the `require-dbt-version` configuration, locally within specific directories using `{dbtenv.LOCAL_VERSION_FILE}` files, or globally in a `{dbtenv.GLOBAL_VERSION_FILE}` file. @@ -56,8 +57,18 @@ def build_common_args_parser(env: Environment, dest_prefix: str = '') -> argpars const=True, help=f""" Output debug information as dbtenv runs. - The default is to not output debug information, but that can be overridden by setting a {dbtenv.DEBUG_VAR} - environment variable. + This can also be enabled by setting a {dbtenv.DEBUG_VAR} environment variable. + """ + ) + common_args_parser.add_argument( + '--quiet', + dest=f'{dest_prefix}quiet', + action='store_const', + const=True, + help=f""" + Don't output any nonessential information as dbtenv runs. + This can also be enabled by setting a {dbtenv.QUIET_VAR} environment variable. + Note that if outputting debug information has been enabled this setting will have no effect. """ ) if env.homebrew_installed: @@ -134,6 +145,10 @@ def main(args: List[str] = None) -> None: logger.debug(f"Parsed arguments = {parsed_args}") + quiet = parsed_args.quiet or parsed_args.get('subcommand_quiet') + if quiet: + env.quiet = quiet + installer = parsed_args.get('installer') or parsed_args.get('subcommand_installer') if installer: env.installer = installer diff --git a/dbtenv/pip.py b/dbtenv/pip.py index 4991d56..5031586 100644 --- a/dbtenv/pip.py +++ b/dbtenv/pip.py @@ -1,4 +1,5 @@ # Standard library +from datetime import date import http import http.server import json @@ -23,7 +24,12 @@ def get_installed_pip_dbt_versions(env: Environment) -> List[Version]: if not os.path.isdir(env.venvs_directory): return [] with os.scandir(env.venvs_directory) as venvs_dir_scan: - possible_versions = (Version(entry.name) for entry in venvs_dir_scan if entry.is_dir()) + venvs_prefix_length = len(env.venvs_prefix) + possible_versions = ( + Version(entry.name[venvs_prefix_length:]) + for entry in venvs_dir_scan + if entry.is_dir() and entry.name.startswith(env.venvs_prefix) + ) return [version for version in possible_versions if PipDbt(env, version).is_installed()] @@ -32,7 +38,7 @@ class PipDbt(Dbt): def __init__(self, env: Environment, version: Version) -> None: super().__init__(env, version) - self.venv_directory = os.path.join(env.venvs_directory, version.pypi_version) + self.venv_directory = os.path.join(env.venvs_directory, f'{env.venvs_prefix}{version.pypi_version}') self._executable: Optional[str] = None def install(self, force: bool = False, package_location: Optional[str] = None, editable: bool = False) -> None: @@ -52,6 +58,8 @@ def install(self, force: bool = False, package_location: Optional[str] = None, e raise DbtenvError(f"Failed to create virtual environment in `{self.venv_directory}`.") pip = self._find_pip() + # Upgrade pip to avoid problems with packages that might require newer pip features. + subprocess.run([pip, 'install', '--upgrade', 'pip']) # Install wheel to avoid pip falling back to using legacy `setup.py` installs. subprocess.run([pip, 'install', '--disable-pip-version-check', 'wheel']) pip_args = ['install', '--disable-pip-version-check'] @@ -64,7 +72,7 @@ def install(self, force: bool = False, package_location: Optional[str] = None, e package_source = "the Python Package Index" if self.env.simulate_release_date: package_metadata = get_pypi_package_metadata('dbt') - release_date = package_metadata['releases'][self.version.pypi_version][0]['upload_time'][:10] + release_date = date.fromisoformat(package_metadata['releases'][self.version.pypi_version][0]['upload_time'][:10]) logger.info(f"Simulating release date {release_date} for dbt {self.version}.") class ReleaseDateFilterPyPIRequestHandler(BaseDateFilterPyPIRequestHandler): date = release_date @@ -152,7 +160,7 @@ def execute(self, args: List[str]) -> None: if broken_python_symlinks: broken_symlink = broken_python_symlinks[0] broken_symlink_target = os.readlink(broken_symlink.path) - logger.warning( + logger.error( f"The virtual environment for dbt {self.version.pypi_version} is broken because the" f" `{broken_symlink.path}` symlink points to `{broken_symlink_target}`, which no longer exists." ) @@ -222,7 +230,7 @@ def do_GET(self) -> None: file['filename'] for files in package_metadata['releases'].values() for file in files - if file['upload_time'][:10] > self.date + if date.fromisoformat(file['upload_time'][:10]) > self.date ) file_link_pattern = r']+>(?P[^<]+)' excluded_file_link_count = 0 diff --git a/dbtenv/version.py b/dbtenv/version.py index ebddb06..bbd6565 100644 --- a/dbtenv/version.py +++ b/dbtenv/version.py @@ -9,7 +9,7 @@ # Local import dbtenv -from dbtenv import Args, DbtenvError, Environment, Subcommand, Version +from dbtenv import Args, Environment, Subcommand, Version import dbtenv.install import dbtenv.versions @@ -84,7 +84,7 @@ def execute(self, args: Args) -> None: else: local_version = try_get_local_version(self.env) if local_version: - print(f"{local_version} (set by {local_version.source})") + print(f"{local_version} ({local_version.source_description})") else: logger.info(f"No local dbt version has been set for `{self.env.working_directory}` using `{dbtenv.LOCAL_VERSION_FILE}` files.") elif args.project_dbt_version is not None: @@ -97,10 +97,10 @@ def execute(self, args: Args) -> None: if project_version: if preferred_version and project_version == preferred_version: logger.info( - f"Preferred version {preferred_version} (set by {preferred_version.source}) is compatible" + f"Preferred version {preferred_version} ({preferred_version.source_description}) is compatible" " with all version requirements in the dbt project." ) - print(f"{project_version} (set by {project_version.source})") + print(f"{project_version} ({project_version.source_description})") # If no project version could be determined, try_get_project_version() will have already logged the reason why. else: logger.error("No dbt project found.") @@ -111,11 +111,8 @@ def execute(self, args: Args) -> None: else: logger.info(f"No dbt version has been set for the current shell using a {dbtenv.DBT_VERSION_VAR} environment variable.") else: - version = try_get_version(self.env) - if version: - print(f"{version} (set by {version.source})") - else: - logger.info("No dbt version has been set for the current shell, dbt project, local directory, or globally.") + version = get_version(self.env) + print(f"{version} ({version.source_description})") def read_version_file(file_path: str) -> Version: @@ -164,7 +161,7 @@ def try_get_shell_version(env: Environment) -> Optional[Version]: class VersionRequirement: def __init__(self, requirement: str, source: str) -> None: self.requirement = requirement - requirement_match = re.match(r'(?P[<>]=?)?(?P.+)', requirement) + requirement_match = re.match(r'(?P[<>=]=?)?(?P.+)', requirement) self.operator = requirement_match['operator'] or '==' self.version = Version(requirement_match['version']) self.source = source @@ -190,7 +187,11 @@ def try_get_project_version_requirements(project_file: str) -> List[VersionRequi with open(project_file) as file: project_file_text = file.read() - requirements_match = re.search(r'^require-dbt-version: *(?P\[[^\]]+\]|\S+)', project_file_text, re.MULTILINE) + requirements_match = re.search( + r'''^require-dbt-version: *(?P\[[^\]]+\]|"[^"]+"|'[^']+'|\S+)''', + project_file_text, + re.MULTILINE + ) if requirements_match: requirements_text = requirements_match['requirements'] logger.debug(f"Found dbt version requirements in `{project_file}`: {requirements_text}") @@ -208,39 +209,53 @@ def try_get_project_version_requirements(project_file: str) -> List[VersionRequi return [] +def get_max_version(versions: Collection[Version]) -> Version: + stable_versions = [version for version in versions if version.is_stable] + if stable_versions: + return sorted(stable_versions)[-1] + else: + return sorted(versions)[-1] + + def try_get_max_compatible_version(versions: Collection[Version], requirements: Collection[VersionRequirement]) -> Optional[Version]: compatible_versions = [ version for version in versions if version.is_semantic and all(requirement.is_compatible_with(version) for requirement in requirements) ] - if not compatible_versions: - return None - - compatible_stable_versions = [version for version in compatible_versions if version.is_stable] - if compatible_stable_versions: - compatible_stable_versions.sort() - return compatible_stable_versions[-1] + if compatible_versions: + return get_max_version(compatible_versions) else: - compatible_versions.sort() - return compatible_versions[-1] + return None def try_get_project_version(env: Environment, preferred_version: Optional[Version] = None) -> Optional[Version]: - version_requirements = try_get_project_version_requirements(env.project_file) - requirements_project_files = [os.path.basename(env.project_file)] + project_version_requirements = try_get_project_version_requirements(env.project_file) + if not project_version_requirements: + logger.debug(f"The dbt project has no version requirements.") + return None + + all_version_requirements = project_version_requirements.copy() + project_file = os.path.basename(env.project_file) + requirements_project_files = [project_file] + scope_decription = "the dbt project" + has_packages_with_version_requirements = False for package_project_file in glob.glob(os.path.join(env.project_directory, 'dbt_modules', '*', 'dbt_project.yml')): package_version_requirements = try_get_project_version_requirements(package_project_file) if package_version_requirements: - version_requirements.extend(package_version_requirements) + all_version_requirements.extend(package_version_requirements) requirements_project_files.append(os.path.relpath(package_project_file, env.project_directory)) + has_packages_with_version_requirements = True + + if has_packages_with_version_requirements: + scope_decription += " and its installed packages" if preferred_version: - for requirement in version_requirements: + for requirement in all_version_requirements: if not requirement.is_compatible_with(preferred_version): logger.info( - f"Preferred version {preferred_version} (set by {preferred_version.source}) is incompatible with" + f"Preferred version {preferred_version} ({preferred_version.source_description}) is incompatible with" f" the {requirement} requirement in `{os.path.relpath(requirement.source, env.project_directory)}`." ) break @@ -248,20 +263,37 @@ def try_get_project_version(env: Environment, preferred_version: Optional[Versio return preferred_version installed_versions = dbtenv.versions.get_installed_versions(env) - compatible_version = try_get_max_compatible_version(installed_versions, version_requirements) + compatible_version = try_get_max_compatible_version(installed_versions, all_version_requirements) if compatible_version: return Version(compatible_version.raw_version, source=', '.join(requirements_project_files)) - else: - logger.info("No installed versions are compatible with all version requirements in the dbt project.") installable_versions = dbtenv.versions.get_installable_versions(env) - compatible_version = try_get_max_compatible_version(installable_versions, version_requirements) + compatible_version = try_get_max_compatible_version(installable_versions, all_version_requirements) if compatible_version: - logger.info(f"{compatible_version} is the latest installable version that is compatible with all version requirements in the dbt project.") + logger.info(f"{compatible_version} is the latest installable version that is compatible with all version requirements in {scope_decription}.") return Version(compatible_version.raw_version, source=', '.join(requirements_project_files)) - else: - logger.warning("No installable versions are compatible with all version requirements in the dbt project.") - return None + + warning = f"No available versions are compatible with all version requirements in {scope_decription}." + if has_packages_with_version_requirements: + warning += " You may need to upgrade installed packages by updating `packages.yml` and running dbt's `deps` sub-command." + logger.warning(warning) + + if has_packages_with_version_requirements: + logger.debug("Trying to get dbt version for the project again while ignoring installed packages in case they're out of date.") + + if preferred_version and all(requirement.is_compatible_with(preferred_version) for requirement in project_version_requirements): + return preferred_version + + compatible_version = try_get_max_compatible_version(installed_versions, project_version_requirements) + if compatible_version: + return Version(compatible_version.raw_version, source=project_file) + + compatible_version = try_get_max_compatible_version(installable_versions, project_version_requirements) + if compatible_version: + logger.info(f"{compatible_version} is the latest installable version that is compatible with all version requirements in the dbt project.") + return Version(compatible_version.raw_version, source=project_file) + + return None def get_version(env: Environment) -> Version: @@ -286,12 +318,11 @@ def get_version(env: Environment) -> Version: if preferred_version: return preferred_version - raise DbtenvError("No dbt version has been set for the current shell, dbt project, local directory, or globally.") - + installed_versions = dbtenv.versions.get_installed_versions(env) + if installed_versions: + max_installed_version = get_max_version(installed_versions) + return Version(max_installed_version.raw_version, source_description="max installed version") -def try_get_version(env: Environment) -> Optional[Version]: - try: - return get_version(env) - except Exception as error: - logger.debug(f"Error getting version: {error}") - return None + installable_versions = dbtenv.versions.get_installable_versions(env) + max_installable_version = get_max_version(installable_versions) + return Version(max_installable_version.raw_version, source_description="max installable version") diff --git a/dbtenv/versions.py b/dbtenv/versions.py index 2654c6e..01e87d8 100644 --- a/dbtenv/versions.py +++ b/dbtenv/versions.py @@ -43,14 +43,14 @@ def execute(self, args: Args) -> None: versions.sort() if versions: - active_version = dbtenv.version.try_get_version(self.env) + active_version = dbtenv.version.get_version(self.env) logger.info("+ = installed, * = active") for version in versions: line = "+ " if version in installed_versions else " " line += "* " if version == active_version else " " line += version.get_installer_version(self.env.primary_installer) if version == active_version: - line += f" (set by {active_version.source})" + line += f" ({active_version.source_description})" print(line) else: logger.info(f"No dbt installations found.") diff --git a/dbtenv/which.py b/dbtenv/which.py index 14b4e5e..d315e67 100644 --- a/dbtenv/which.py +++ b/dbtenv/which.py @@ -47,14 +47,10 @@ def execute(self, args: Args) -> None: if args.dbt_version: version = args.dbt_version else: - version = dbtenv.version.try_get_version(self.env) - if version: - logger.info(f"Using dbt {version} (set by {version.source}).") - else: - logger.info("No dbt version has been set for the current shell, dbt project, local directory, or globally.") + version = dbtenv.version.get_version(self.env) + logger.info(f"Using dbt {version} ({version.source_description}).") - if version: - print(get_dbt(self.env, version).get_executable()) + print(get_dbt(self.env, version).get_executable()) def get_dbt(env: Environment, version: Version) -> Dbt: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5182107 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=51", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..554f3a6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,26 @@ +[metadata] +name = dbtenv +version = attr: dbtenv.__version__ +description = dbtenv lets you easily install and run multiple versions of dbt using pip with Python virtual environments, or optionally using Homebrew on Mac or Linux. +long_description = file: README.md +long_description_content_type = text/markdown +author = Brooklyn Data Co. +author_email = hello@brooklyndata.co +url = https://github.com/brooklyn-data/dbtenv +classifiers = + Development Status :: 5 - Production/Stable + License :: OSI Approved :: Apache Software License + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft :: Windows + Operating System :: POSIX :: Linux + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +packages = dbtenv +python_requires = >=3.7 + +[options.entry_points] +console_scripts = + dbtenv = dbtenv.main:main diff --git a/setup.py b/setup.py deleted file mode 100644 index 8d60a5e..0000000 --- a/setup.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Standard library -from distutils.core import setup - -# Local -import dbtenv - - -description = ( - "dbtenv lets you easily install and run multiple versions of dbt using pip with Python virtual environments," - " or optionally using Homebrew on Mac or Linux." -) - -setup( - name='dbtenv', - version=dbtenv.__version__, - description=description, - long_description=description, - author='Brooklyn Data Co.', - author_email='hello@brooklyndata.co', - url='https://github.com/brooklyn-data/dbtenv', - packages=['dbtenv'], - entry_points={ - 'console_scripts': [ - 'dbtenv = dbtenv.main:main' - ] - }, - python_requires='>=3.6.3', - classifiers=[ - 'Development Status :: 4 - Beta', - - 'License :: OSI Approved :: Apache Software License', - - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX :: Linux', - - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ] -)