Skip to content

Commit

Permalink
Merge pull request #23 from brooklyn-data/2.0.0a2
Browse files Browse the repository at this point in the history 
Release 2.0.0a2
  • Loading branch information
@NiallRees
NiallRees committed Mar 24, 2022
2 parents e4e48ec + 40f8aab commit 56000c3
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 156 deletions.
16 changes: 15 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,28 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased](https://github.com/brooklyn-data/dbtenv/compare/v2.0.0a1...HEAD)
## [Unreleased](https://github.com/brooklyn-data/dbtenv/compare/v2.0.0a2...HEAD)

### Added

### Changed

### Fixed

## [2.0.0a2](https://github.com/brooklyn-data/dbtenv/compare/v2.0.0a1...v2.0.0a2)

### Added
- The execute command's `--dbt` argument can now take either a dbt version (e.g. 1.0.0) or full pip specifier to use (e.g. dbt-snowflake==1.0.0). dbtenv will attempt to automatically detect the required adapter or version from the environment if not specified.

### Changed
- Attempting to install a version of dbt which doesn't exist will exit cleanly, and provide a list of available versions for that adapter.
- Failed dbt version installations exit cleanly, removing the created virtual environment.
- Improved logging.

### Fixed
- Only entries in the environment directory which are dbtenv 2.0.0 environments will be read as installed dbt versions, fixing an issue where dbtenv 1.0.0 environments caused a failure.
- Fixed version command, and all dbtenv config files. These can now take either a dbt version (e.g. 1.0.0) or full pip specifier to use (e.g. dbt-snowflake==1.0.0). dbtenv will attempt to automatically detect the required adapter or version from the environment if not specified.

## [2.0.0a1](https://github.com/brooklyn-data/dbtenv/compare/v1.3.2...v2.0.0a1)

### Added
Expand Down
5 changes: 3 additions & 2 deletions dbtenv/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import os.path
import platform
import re
import shutil
import subprocess
import sys
from typing import Any, List, Optional
Expand Down Expand Up @@ -73,9 +72,11 @@ class Version(distutils.version.LooseVersion):
def __init__(self, pip_specifier: str = None, adapter_type: str = None, version: str = None, source: Optional[str] = None, source_description: Optional[str] = None) -> None:
if pip_specifier:
self.pip_specifier = pip_specifier
self.name, self.version = re.match(r"(.*)==(.*)", pip_specifier).groups()
self.name, self.version = re.match(r"^(dbt-.+)==(.+)$", pip_specifier).groups()
self.adapter_type = self.name.replace("dbt-", "")
self.pypi_version = self.version
else:
self.adapter_type = adapter_type
self.name = f"dbt-{adapter_type}"
self.pypi_version = version
self.pip_specifier = f"{self.name}=={self.pypi_version}"
Expand Down
33 changes: 21 additions & 12 deletions dbtenv/execute.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@ def add_args_parser(self, subparsers: argparse._SubParsersAction, parent_parsers
)
parser.add_argument(
'--dbt',
dest='dbt_version',
dest='dbt_version_specifier',
type=str,
metavar='<dbt_version>',
metavar='<dbt_version_specifier>',
help="""
dbt version to use (e.g. 1.0.1).
If not specified, the dbt version will be automatically detected from the environment.
dbt version (e.g. 1.0.0) or full pip specifier to use (e.g. dbt-snowflake==1.0.0).
dbtenv will attempt to automatically detect the required adapter or version from the environment if not specified.
"""
)
parser.add_argument(
Expand All @@ -59,17 +59,26 @@ def execute(self, args: Args) -> None:
if arg == "--target":
arg_target_name = args.dbt_args[i+1]
break
adapter_type = dbtenv.version.try_get_project_adapter_type(self.env.project_file, target_name=arg_target_name)
if not adapter_type:
logger.info("Could not determine adapter, either not running inside dbt project or no default target is set for the current project in profiles.yml.")
return

if args.dbt_version:
version = Version(adapter_type=adapter_type, version=args.dbt_version)
adapter_type = dbtenv.version.try_get_project_adapter_type(self.env.project_file, target_name=arg_target_name)
if args.dbt_version_specifier:
if bool(re.search(r"^(dbt-.+)==(.+)$", args.dbt_version_specifier)):
version = Version(pip_specifier=args.dbt_version_specifier, source_description="specified using --dbt arg")
elif bool(re.search(r"^[0-9\.]+[a-z0-9]*$", args.dbt_version_specifier)):
if not adapter_type:
logger.info("Could not determine adapter type as no default target is set for the current project in profiles.yml.")
return
version = Version(adapter_type=adapter_type, version=args.dbt_version_specifier, source_description="adapter type automatically detected, version specified using --dbt arg")
else:
logger.info("The argument passed to --dbt didn't match a dbt version (e.g. 1.0.0) or full pip specifier (e.g. dbt-snowflake==1.0.0).")
return
else:
arg_target_name = None
version = dbtenv.version.get_version(self.env, adapter_type=adapter_type)
logger.info(f"Using {version} ({version.source_description}).")

if not version:
return

logger.info(f"Using {version} ({version.source_description}).")

dbt = dbtenv.which.try_get_dbt(self.env, version)
if not dbt:
Expand Down
121 changes: 65 additions & 56 deletions dbtenv/pip.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,31 @@ def get_installed_pip_dbt_versions(env: Environment, adapter_type: Optional[str]
return []
versions = []
for entry in os.scandir(env.venvs_directory):
if not adapter_type or entry.name.startswith(f"dbt-{adapter_type}"):
if entry.is_dir() and bool(re.search(r"^(dbt-.+)==(.+)$", entry.name)) and (not adapter_type or entry.name.startswith(f"dbt-{adapter_type}==")):
versions.append(
Version(pip_specifier=entry.name)
)
return versions


def get_pypi_package_metadata(package: str) -> str:
package_json_url = f'https://pypi.org/pypi/{package}/json'
logger.debug(f"Fetching {package} package metadata from {package_json_url}.")
with urllib.request.urlopen(package_json_url) as package_json_response:
return json.load(package_json_response)

def get_pypi_package_versions(adapter_type: str) -> List[Version]:
package_metadata = get_pypi_package_metadata(f"dbt-{adapter_type}")
possible_versions = ((Version(adapter_type=adapter_type, version=version), files) for version, files in package_metadata['releases'].items())
return [version for version, files in possible_versions if any(not file['yanked'] for file in files)]

def get_pypi_all_dbt_package_versions() -> List[Version]:
versions = []
for adapter_type in DBT_ADAPTER_TYPES:
versions += get_pypi_package_versions(adapter_type)
return versions


class PipDbt(Dbt):
"""A specific version of dbt installed with pip in a Python virtual environment."""

Expand All @@ -78,6 +96,11 @@ def install(self, force: bool = False, package_location: Optional[str] = None, e
else:
raise DbtenvError(f"`{self.venv_directory}` already exists.")

available_adapter_versions = get_pypi_package_versions(adapter_type=self.version.adapter_type)
if self.version not in available_adapter_versions:
logger.info(f"{self.version} is not available for installation from pypi. Try one of {[v.pypi_version for v in available_adapter_versions]}")
return

python = self.env.python
self._check_python_compatibility(python)

Expand All @@ -86,43 +109,47 @@ def install(self, force: bool = False, package_location: Optional[str] = None, e
if venv_result.returncode != 0:
raise DbtenvError(f"Failed to create virtual environment in `{self.venv_directory}`.")

pip = self._find_pip()
# Upgrade pip to avoid problems with packages that might require newer pip features.
subprocess.run([pip, 'install', '--upgrade', 'pip'])
# Install wheel to avoid pip falling back to using legacy `setup.py` installs.
subprocess.run([pip, 'install', '--disable-pip-version-check', 'wheel'])
pip_args = ['install', '--disable-pip-version-check']
if package_location:
package_source = f"`{package_location}`"
if editable:
pip_args.append('--editable')
pip_args.append(package_location)
else:
package_source = "the Python Package Index"
if self.env.simulate_release_date:
package_metadata = get_pypi_package_metadata('dbt')
release_date = date.fromisoformat(package_metadata['releases'][self.version.pypi_version][0]['upload_time'][:10])
logger.info(f"Simulating release date {release_date} for dbt {self.version}.")
class ReleaseDateFilterPyPIRequestHandler(BaseDateFilterPyPIRequestHandler):
date = release_date
pip_filter_server = http.server.HTTPServer(('', 0), ReleaseDateFilterPyPIRequestHandler)
pip_filter_port = pip_filter_server.socket.getsockname()[1]
threading.Thread(target=pip_filter_server.serve_forever, daemon=True).start()
pip_args.extend(['--index-url', f'http://localhost:{pip_filter_port}/simple'])
elif self.version.pypi_version < '0.19.1':
# Versions prior to 0.19.1 just specified agate>=1.6, but agate 1.6.2 introduced a dependency on PyICU
# which causes installation problems, so exclude that like versions 0.19.1 and above do.
pip_args.append('agate>=1.6,<1.6.2')

pip_args.append(self.version.pip_specifier)
logger.info(f"Installing {self.version.pip_specifier} from {package_source} into `{self.venv_directory}`.")

logger.debug(f"Running `{pip}` with arguments {pip_args}.")
pip_result = subprocess.run([pip, *pip_args])
if pip_result.returncode != 0:
raise DbtenvError(f"Failed to install dbt {self.version.pypi_version} from {package_source} into `{self.venv_directory}`.")

logger.info(f"Successfully installed dbt {self.version.pypi_version} from {package_source} into `{self.venv_directory}`.")
try:
pip = self._find_pip()
# Upgrade pip to avoid problems with packages that might require newer pip features.
subprocess.run([pip, 'install', '--upgrade', 'pip'])
# Install wheel to avoid pip falling back to using legacy `setup.py` installs.
subprocess.run([pip, 'install', '--disable-pip-version-check', 'wheel'])
pip_args = ['install', '--disable-pip-version-check']
if package_location:
package_source = f"`{package_location}`"
if editable:
pip_args.append('--editable')
pip_args.append(package_location)
else:
package_source = "the Python Package Index"
if self.env.simulate_release_date:
package_metadata = get_pypi_package_metadata('dbt')
release_date = date.fromisoformat(package_metadata['releases'][self.version.pypi_version][0]['upload_time'][:10])
logger.info(f"Simulating release date {release_date} for dbt {self.version}.")
class ReleaseDateFilterPyPIRequestHandler(BaseDateFilterPyPIRequestHandler):
date = release_date
pip_filter_server = http.server.HTTPServer(('', 0), ReleaseDateFilterPyPIRequestHandler)
pip_filter_port = pip_filter_server.socket.getsockname()[1]
threading.Thread(target=pip_filter_server.serve_forever, daemon=True).start()
pip_args.extend(['--index-url', f'http://localhost:{pip_filter_port}/simple'])
elif self.version.pypi_version < '0.19.1':
# Versions prior to 0.19.1 just specified agate>=1.6, but agate 1.6.2 introduced a dependency on PyICU
# which causes installation problems, so exclude that like versions 0.19.1 and above do.
pip_args.append('agate>=1.6,<1.6.2')

pip_args.append(self.version.pip_specifier)
logger.info(f"Installing {self.version.pip_specifier} from {package_source} into `{self.venv_directory}`.")

logger.debug(f"Running `{pip}` with arguments {pip_args}.")
pip_result = subprocess.run([pip, *pip_args])
if pip_result.returncode != 0:
raise DbtenvError(f"Failed to install dbt {self.version.pypi_version} from {package_source} into `{self.venv_directory}`.")
except Exception as e:
shutil.rmtree(self.venv_directory)
raise(e)

logger.info(f"Successfully installed {self.version} from {package_source} into `{self.venv_directory}`.")

def _check_python_compatibility(self, python: str) -> None:
python_version_result = subprocess.run([python, '--version'], stdout=subprocess.PIPE)
Expand Down Expand Up @@ -209,24 +236,6 @@ def uninstall(self, force: bool = False) -> None:
logger.info(f"Successfully uninstalled dbt {self.version.pypi_version} from `{self.venv_directory}`.")


def get_pypi_package_metadata(package: str) -> str:
package_json_url = f'https://pypi.org/pypi/{package}/json'
logger.debug(f"Fetching {package} package metadata from {package_json_url}.")
with urllib.request.urlopen(package_json_url) as package_json_response:
return json.load(package_json_response)

def get_pypi_package_versions(adapter_type: str) -> List[Version]:
package_metadata = get_pypi_package_metadata(f"dbt-{adapter_type}")
possible_versions = ((Version(adapter_type=adapter_type, version=version), files) for version, files in package_metadata['releases'].items())
return [version for version, files in possible_versions if any(not file['yanked'] for file in files)]

def get_pypi_all_dbt_package_versions() -> List[Version]:
versions = []
for adapter_type in DBT_ADAPTER_TYPES:
versions += get_pypi_package_versions(adapter_type)
return versions


class BaseDateFilterPyPIRequestHandler(http.server.BaseHTTPRequestHandler):
"""
HTTP request handler that proxies PEP 503-compliant requests to pypi.org and excludes files uploaded after self.date.
Expand Down
Loading

0 comments on commit 56000c3

Please sign in to comment.