From 187b3ed79512e776a47a308b5dd6ee5e2cd5d1a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Feb 2022 13:31:44 +0100 Subject: [PATCH] Release 0.14.0 (#55) * feat: execution env config --- CHANGELOG.md | 6 +++- data_pipelines_cli/__init__.py | 2 +- data_pipelines_cli/cli_commands/compile.py | 14 +++----- data_pipelines_cli/cli_commands/create.py | 4 +-- data_pipelines_cli/cli_commands/deploy.py | 4 +-- data_pipelines_cli/cli_commands/publish.py | 24 ++++--------- data_pipelines_cli/cli_commands/template.py | 4 +-- data_pipelines_cli/data_structures.py | 8 ++--- data_pipelines_cli/dbt_utils.py | 4 +-- data_pipelines_cli/docker_response_reader.py | 24 ++++--------- data_pipelines_cli/errors.py | 12 ++----- data_pipelines_cli/filesystem_utils.py | 4 +-- data_pipelines_cli/io_utils.py | 11 ++---- data_pipelines_cli/jinja.py | 8 ++--- docs/usage.rst | 3 ++ pyproject.toml | 2 ++ setup.cfg | 4 +-- setup.py | 2 +- tests/cli_commands/test_clean.py | 4 +-- tests/cli_commands/test_compile.py | 32 +++++------------ tests/cli_commands/test_create.py | 8 ++--- tests/cli_commands/test_deploy.py | 36 +++++--------------- tests/cli_commands/test_prepare_env.py | 36 +++++--------------- tests/cli_commands/test_publish.py | 24 ++++--------- tests/goldens/config/base/execution_env.yml | 5 +++ tests/goldens/config/base/k8s.yml | 3 -- tests/test_cli_utils.py | 20 +++-------- tests/test_config_generation.py | 12 ++----- tests/test_data_structures.py | 8 ++--- tests/test_filesystem_utils.py | 24 ++++--------- tests/test_io_utils.py | 4 +-- tox.ini | 2 +- 32 files changed, 106 insertions(+), 252 deletions(-) create mode 100644 tests/goldens/config/base/execution_env.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index ca7a254..638588e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [0.14.0] - 2022-02-02 + ## [0.13.0] - 2022-02-01 ## [0.12.0] - 2022-01-31 @@ -150,7 +152,9 @@ - Draft of `dp init`, `dp create`, `dp template new`, `dp template list` and `dp dbt` - Draft of `dp compile` and `dp deploy` -[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.13.0...HEAD +[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.14.0...HEAD + +[0.14.0]: https://github.com/getindata/data-pipelines-cli/compare/0.13.0...0.14.0 [0.13.0]: https://github.com/getindata/data-pipelines-cli/compare/0.12.0...0.13.0 diff --git a/data_pipelines_cli/__init__.py b/data_pipelines_cli/__init__.py index a82a381..b3eea6a 100644 --- a/data_pipelines_cli/__init__.py +++ b/data_pipelines_cli/__init__.py @@ -5,4 +5,4 @@ pipelines. """ -version = "0.13.0" +version = "0.14.0" diff --git a/data_pipelines_cli/cli_commands/compile.py b/data_pipelines_cli/cli_commands/compile.py index 560a93b..7a494f5 100644 --- a/data_pipelines_cli/cli_commands/compile.py +++ b/data_pipelines_cli/cli_commands/compile.py @@ -66,18 +66,14 @@ def _copy_dbt_manifest() -> None: ) -def _replace_k8s_settings(docker_args: DockerArgs) -> None: - k8s_config = BUILD_DIR.joinpath("dag", "config", "base", "k8s.yml") - echo_info( - f"Replacing {IMAGE_TAG_TO_REPLACE} with commit SHA = {docker_args.commit_sha}" - ) +def replace_image_settings(docker_args: DockerArgs) -> None: + k8s_config = BUILD_DIR.joinpath("dag", "config", "base", "execution_env.yml") + echo_info(f"Replacing {IMAGE_TAG_TO_REPLACE} with commit SHA = {docker_args.commit_sha}") replace(k8s_config, IMAGE_TAG_TO_REPLACE, docker_args.commit_sha) def _replace_datahub_with_jinja_vars(env: str) -> None: - datahub_config_path: pathlib.Path = BUILD_DIR.joinpath( - "dag", "config", "base", "datahub.yml" - ) + datahub_config_path: pathlib.Path = BUILD_DIR.joinpath("dag", "config", "base", "datahub.yml") if not datahub_config_path.exists(): echo_warning( @@ -112,7 +108,7 @@ def compile_project( copy_config_dir_to_build_dir() docker_args = DockerArgs(env) - _replace_k8s_settings(docker_args) + replace_image_settings(docker_args) _replace_datahub_with_jinja_vars(env) _dbt_compile(env) diff --git a/data_pipelines_cli/cli_commands/create.py b/data_pipelines_cli/cli_commands/create.py index a2238fa..46a2791 100644 --- a/data_pipelines_cli/cli_commands/create.py +++ b/data_pipelines_cli/cli_commands/create.py @@ -65,7 +65,5 @@ def create(project_path: str, template_path: Optional[str]) -> None: @click.argument("template-path", nargs=-1) def create_command(project_path: str, template_path: Sequence[str]) -> None: if template_path and len(template_path) > 1: - echo_warning( - "dp create expects at most two arguments -- project-path and template-path" - ) + echo_warning("dp create expects at most two arguments -- project-path and template-path") create(project_path, template_path[0] if template_path else None) diff --git a/data_pipelines_cli/cli_commands/deploy.py b/data_pipelines_cli/cli_commands/deploy.py index 452aef9..4f94030 100644 --- a/data_pipelines_cli/cli_commands/deploy.py +++ b/data_pipelines_cli/cli_commands/deploy.py @@ -129,9 +129,7 @@ def _sync_bucket(self) -> None: name="deploy", help="Push and deploy the project to the remote machine", ) -@click.option( - "--env", default="base", show_default=True, type=str, help="Name of the environment" -) +@click.option("--env", default="base", show_default=True, type=str, help="Name of the environment") @click.option("--dags-path", required=False, help="Remote storage URI") @click.option( "--blob-args", diff --git a/data_pipelines_cli/cli_commands/publish.py b/data_pipelines_cli/cli_commands/publish.py index c76e3dc..4a3638e 100644 --- a/data_pipelines_cli/cli_commands/publish.py +++ b/data_pipelines_cli/cli_commands/publish.py @@ -34,9 +34,7 @@ def _get_database_and_schema_name(manifest: Manifest) -> Tuple[str, str]: raise DataPipelinesError("There is no model in 'manifest.json' file.") -def _parse_columns_dict_into_table_list( - columns: Dict[str, ColumnInfo] -) -> List[DbtTableColumn]: +def _parse_columns_dict_into_table_list(columns: Dict[str, ColumnInfo]) -> List[DbtTableColumn]: return [ DbtTableColumn( name=column.name, @@ -64,16 +62,12 @@ def _parse_models_schema(manifest: Manifest) -> List[DbtModel]: def _get_dag_id() -> str: - with open( - BUILD_DIR.joinpath("dag", "config", "base", "airflow.yml"), "r" - ) as airflow_yml: + with open(BUILD_DIR.joinpath("dag", "config", "base", "airflow.yml"), "r") as airflow_yml: return yaml.safe_load(airflow_yml)["dag"]["dag_id"] def _create_source(project_name: str) -> DbtSource: - with open( - pathlib.Path.cwd().joinpath("target", "manifest.json"), "r" - ) as manifest_json: + with open(pathlib.Path.cwd().joinpath("target", "manifest.json"), "r") as manifest_json: manifest_dict = json.load(manifest_json) manifest = Manifest.from_dict(manifest_dict) @@ -130,9 +124,7 @@ def _clean_repo(packages_repo: pathlib.Path) -> None: shutil.rmtree(packages_repo) -def _copy_publication_to_repo( - package_dest: pathlib.Path, package_path: pathlib.Path -) -> None: +def _copy_publication_to_repo(package_dest: pathlib.Path, package_path: pathlib.Path) -> None: if package_dest.exists(): echo_info(f"Removing {package_dest}") shutil.rmtree(package_dest) @@ -145,14 +137,10 @@ def _configure_git_env(repo: Repo, config: Dict[str, Any]) -> None: repo.config_writer().set_value("user", "email", config["email"]).release() -def _commit_and_push_changes( - repo: Repo, project_name: str, project_version: str -) -> None: +def _commit_and_push_changes(repo: Repo, project_name: str, project_version: str) -> None: echo_info("Publishing") repo.git.add(all=True) - repo.index.commit( - f"Publication from project {project_name}, version: {project_version}" - ) + repo.index.commit(f"Publication from project {project_name}, version: {project_version}") origin = repo.remote(name="origin") origin.push() diff --git a/data_pipelines_cli/cli_commands/template.py b/data_pipelines_cli/cli_commands/template.py index a285f83..1594edf 100644 --- a/data_pipelines_cli/cli_commands/template.py +++ b/data_pipelines_cli/cli_commands/template.py @@ -13,8 +13,6 @@ def list_templates() -> None: click.echo(yaml.dump(tc)) -@click.command( - name="template-list", help="Print a list of all templates saved in the config file" -) +@click.command(name="template-list", help="Print a list of all templates saved in the config file") def list_templates_command() -> None: list_templates() diff --git a/data_pipelines_cli/data_structures.py b/data_pipelines_cli/data_structures.py index c769b1f..9b5ff6b 100644 --- a/data_pipelines_cli/data_structures.py +++ b/data_pipelines_cli/data_structures.py @@ -91,14 +91,14 @@ def _get_docker_repository_uri_from_k8s_config(env: str) -> str: read_dictionary_from_config_directory, ) - k8s_config = read_dictionary_from_config_directory( - BUILD_DIR.joinpath("dag"), env, "k8s.yml" + execution_env_config = read_dictionary_from_config_directory( + BUILD_DIR.joinpath("dag"), env, "execution_env.yml" ) try: - return k8s_config["image"]["repository"] + return execution_env_config["image"]["repository"] except KeyError as key_error: raise DataPipelinesError( - f"Could not find 'repository' variable in build/config/{env}/k8s.yml." + f"Could not find 'repository' variable in build/config/{env}/execution_env.yml." ) from key_error diff --git a/data_pipelines_cli/dbt_utils.py b/data_pipelines_cli/dbt_utils.py index b48e188..4f7cfd2 100644 --- a/data_pipelines_cli/dbt_utils.py +++ b/data_pipelines_cli/dbt_utils.py @@ -40,9 +40,7 @@ def _dump_dbt_vars_from_configs_to_string(env: str) -> str: return yaml.dump(dbt_vars, default_flow_style=True, width=sys.maxsize) -def run_dbt_command( - command: Tuple[str, ...], env: str, profiles_path: pathlib.Path -) -> None: +def run_dbt_command(command: Tuple[str, ...], env: str, profiles_path: pathlib.Path) -> None: """ Run dbt subprocess in a context of specified *env*. diff --git a/data_pipelines_cli/docker_response_reader.py b/data_pipelines_cli/docker_response_reader.py index eea798d..a3ba489 100644 --- a/data_pipelines_cli/docker_response_reader.py +++ b/data_pipelines_cli/docker_response_reader.py @@ -83,9 +83,7 @@ def click_echo_ok_responses(self) -> None: click.echo(response.msg) @staticmethod - def _prepare_status( - log: Dict[str, Union[str, Dict[str, str]]] - ) -> DockerReadResponse: + def _prepare_status(log: Dict[str, Union[str, Dict[str, str]]]) -> DockerReadResponse: status_message = cast(str, log["status"]) progress_detail = cast(str, log.get("progressDetail", "")) status_id = cast(str, log.get("id", "")) @@ -98,9 +96,7 @@ def _prepare_status( return DockerReadResponse(message, False) @staticmethod - def _prepare_stream( - log: Dict[str, Union[str, Dict[str, str]]] - ) -> List[DockerReadResponse]: + def _prepare_stream(log: Dict[str, Union[str, Dict[str, str]]]) -> List[DockerReadResponse]: stream = cast(str, log["stream"]) return list( map( @@ -110,9 +106,7 @@ def _prepare_stream( ) @staticmethod - def _prepare_aux( - log: Dict[str, Union[str, Dict[str, str]]] - ) -> List[DockerReadResponse]: + def _prepare_aux(log: Dict[str, Union[str, Dict[str, str]]]) -> List[DockerReadResponse]: aux = cast(Dict[str, str], log["aux"]) to_return = [] if "Digest" in aux: @@ -122,21 +116,15 @@ def _prepare_aux( return to_return @staticmethod - def _prepare_error_detail( - log: Dict[str, Union[str, Dict[str, str]]] - ) -> DockerReadResponse: + def _prepare_error_detail(log: Dict[str, Union[str, Dict[str, str]]]) -> DockerReadResponse: error_detail = cast(Dict[str, str], log["errorDetail"]) error_message = error_detail.get("message", "") error_code = error_detail.get("code", None) return DockerReadResponse( - "ERROR: " - + error_message - + (f"\nError code: {error_code}" if error_code else ""), + "ERROR: " + error_message + (f"\nError code: {error_code}" if error_code else ""), True, ) @staticmethod - def _prepare_error( - log: Dict[str, Union[str, Dict[str, str]]] - ) -> DockerReadResponse: + def _prepare_error(log: Dict[str, Union[str, Dict[str, str]]]) -> DockerReadResponse: return DockerReadResponse("ERROR: " + cast(str, log["error"]), True) diff --git a/data_pipelines_cli/errors.py b/data_pipelines_cli/errors.py index d85ecde..ac31fa7 100644 --- a/data_pipelines_cli/errors.py +++ b/data_pipelines_cli/errors.py @@ -22,9 +22,7 @@ class NoConfigFileError(DataPipelinesError): """Exception raised if `.dp.yml` does not exist""" def __init__(self) -> None: - self.message = ( - "`.dp.yml` config file does not exists. Run 'dp init' to create it." - ) + self.message = "`.dp.yml` config file does not exists. Run 'dp init' to create it." class NotAProjectDirectoryError(DataPipelinesError): @@ -41,9 +39,7 @@ class SubprocessNonZeroExitError(DataPipelinesError): """Exception raised if subprocess exits with non-zero exit code""" def __init__(self, subprocess_name: str, exit_code: int) -> None: - self.message = ( - f"{subprocess_name} has exited with non-zero exit code: {exit_code}" - ) + self.message = f"{subprocess_name} has exited with non-zero exit code: {exit_code}" class SubprocessNotFound(DataPipelinesError): @@ -76,9 +72,7 @@ class AirflowDagsPathKeyError(DataPipelinesError): """Exception raised if there is no ``dags_path`` in `airflow.yml` file.""" def __init__(self) -> None: - self.message = ( - "Variable 'dags_path' cannot be found in 'airflow.yml' config file." - ) + self.message = "Variable 'dags_path' cannot be found in 'airflow.yml' config file." class DockerErrorResponseError(DataPipelinesError): diff --git a/data_pipelines_cli/filesystem_utils.py b/data_pipelines_cli/filesystem_utils.py index bfe0b7a..1dd8fa8 100644 --- a/data_pipelines_cli/filesystem_utils.py +++ b/data_pipelines_cli/filesystem_utils.py @@ -29,9 +29,7 @@ def __init__( remote_kwargs: Dict[str, str], ) -> None: if not pathlib.Path(local_path).exists(): - raise DataPipelinesError( - f"{local_path} does not exists. Run 'dp compile' before." - ) + raise DataPipelinesError(f"{local_path} does not exists. Run 'dp compile' before.") self.local_path_str = str(local_path).rstrip("/") self.local_fs = fsspec.filesystem("file") diff --git a/data_pipelines_cli/io_utils.py b/data_pipelines_cli/io_utils.py index c5171df..5b4b331 100644 --- a/data_pipelines_cli/io_utils.py +++ b/data_pipelines_cli/io_utils.py @@ -13,9 +13,7 @@ # Python's `sed` equivalent, based on the following answer: # https://stackoverflow.com/a/31499114 -def replace( - filename: Union[str, os.PathLike[str]], pattern: str, replacement: str -) -> None: +def replace(filename: Union[str, os.PathLike[str]], pattern: str, replacement: str) -> None: """ Perform the pure-Python equivalent of in-place `sed` substitution: e.g., ``sed -i -e 's/'${pattern}'/'${replacement}' "${filename}"``. @@ -49,9 +47,7 @@ def git_revision_hash() -> Optional[str]: :rtype: Optional[str] """ try: - rev_process = subprocess.run( - ["git", "rev-parse", "HEAD"], check=True, capture_output=True - ) + rev_process = subprocess.run(["git", "rev-parse", "HEAD"], check=True, capture_output=True) return rev_process.stdout.decode("ascii").strip() except FileNotFoundError: click.echo( @@ -62,8 +58,7 @@ def git_revision_hash() -> Optional[str]: return None except subprocess.CalledProcessError as err: click.echo( - "The tool has run across a following error when trying to " - "get Git revision hash:", + "The tool has run across a following error when trying to " "get Git revision hash:", file=sys.stderr, ) click.echo(err.stderr, file=sys.stderr) diff --git a/data_pipelines_cli/jinja.py b/data_pipelines_cli/jinja.py index 7b24976..47d41ac 100644 --- a/data_pipelines_cli/jinja.py +++ b/data_pipelines_cli/jinja.py @@ -45,14 +45,10 @@ def replace_vars_with_values( rendered_settings = {} for setting_key, setting_old_value in templated_dictionary.items(): if isinstance(setting_old_value, dict): - rendered_settings[setting_key] = replace_vars_with_values( - setting_old_value, dbt_vars - ) + rendered_settings[setting_key] = replace_vars_with_values(setting_old_value, dbt_vars) else: try: - rendered_settings[setting_key] = jinja_env.get_template( - setting_key - ).render() + rendered_settings[setting_key] = jinja_env.get_template(setting_key).render() except TypeError: # Jinja is accepting only str or Template and fails on int, etc. rendered_settings[setting_key] = setting_old_value diff --git a/docs/usage.rst b/docs/usage.rst index 302633e..10c2b74 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -93,6 +93,9 @@ setting those in your ``config//k8s.yml`` file, in ``envs`` dictionary: project: my-gcp-project threads: 1 + # config/base/execution_env.yml + # ... General config for execution env ... + # config/base/k8s.yml # ... Kubernetes settings ... diff --git a/pyproject.toml b/pyproject.toml index e69de29..037585e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 100 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a7666a2..3d935b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.13.0 +current_version = 0.14.0 [bumpversion:file:setup.py] @@ -7,7 +7,7 @@ current_version = 0.13.0 [flake8] exclude = .git,__pycache__,build,dist,docs/source/conf.py -max-line-length = 88 +max-line-length = 100 extend-ignore = E203 [mypy] diff --git a/setup.py b/setup.py index 5c46df1..6fcbcac 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( name="data_pipelines_cli", - version="0.13.0", + version="0.14.0", description="CLI for data platform", long_description=README, long_description_content_type="text/markdown", diff --git a/tests/cli_commands/test_clean.py b/tests/cli_commands/test_clean.py index c53174e..588e337 100644 --- a/tests/cli_commands/test_clean.py +++ b/tests/cli_commands/test_clean.py @@ -17,9 +17,7 @@ def _mock_run(self, args: List[str]): self.subprocess_run_args = args def test_clean(self): - with patch( - "data_pipelines_cli.cli_commands.clean.subprocess_run", self._mock_run - ): + with patch("data_pipelines_cli.cli_commands.clean.subprocess_run", self._mock_run): runner = CliRunner() result = runner.invoke(_cli, ["clean"]) self.assertEqual(0, result.exit_code, msg=result.exception) diff --git a/tests/cli_commands/test_compile.py b/tests/cli_commands/test_compile.py index 33b45a0..bc9d261 100644 --- a/tests/cli_commands/test_compile.py +++ b/tests/cli_commands/test_compile.py @@ -19,17 +19,13 @@ class CompileCommandTestCase(unittest.TestCase): @staticmethod - def _k8s_content(repository_url: str, tag: str): + def _execution_env_content(repository_url: str, tag: str): return { "image": { "repository": repository_url, "tag": tag, }, - "variable1": 1337, - "var2": "Hello, world!", - "envs": { - "SOME_BOOLEAN": True, - }, + "type": "k8s", } def setUp(self) -> None: @@ -47,9 +43,7 @@ def test_no_args(self, mock_git_revision_hash): runner = CliRunner() with tempfile.TemporaryDirectory() as tmp_dir, patch( "data_pipelines_cli.cli_commands.compile.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( - "data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( + ), patch("data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir)), patch( "data_pipelines_cli.cli_constants.BUILD_DIR", pathlib.Path(tmp_dir) ), patch( "data_pipelines_cli.dbt_utils.BUILD_DIR", pathlib.Path(tmp_dir) @@ -69,27 +63,21 @@ def test_no_args(self, mock_git_revision_hash): self.assertIn("dbt source freshness", args_str) tmp_dir_path = pathlib.Path(tmp_dir) - with open( - tmp_dir_path.joinpath("dag", "manifest.json"), "r" - ) as tmp_manifest, open( + with open(tmp_dir_path.joinpath("dag", "manifest.json"), "r") as tmp_manifest, open( goldens_dir_path.joinpath("target", "manifest.json"), "r" ) as golden_manifest: - self.assertDictEqual( - json.load(golden_manifest), json.load(tmp_manifest) - ) + self.assertDictEqual(json.load(golden_manifest), json.load(tmp_manifest)) with open( tmp_dir_path.joinpath("dag", "config", "base", "datahub.yml"), "r" ) as tmp_datahub, open( goldens_dir_path.joinpath("config", "base", "datahub.yml") ) as golden_datahub: - self.assertDictEqual( - yaml.safe_load(golden_datahub), yaml.safe_load(tmp_datahub) - ) + self.assertDictEqual(yaml.safe_load(golden_datahub), yaml.safe_load(tmp_datahub)) with open( - tmp_dir_path.joinpath("dag", "config", "base", "k8s.yml"), "r" + tmp_dir_path.joinpath("dag", "config", "base", "execution_env.yml"), "r" ) as tmp_k8s: self.assertDictEqual( - self._k8s_content("my_docker_repository_uri", "aaa9876aaa"), + self._execution_env_content("my_docker_repository_uri", "aaa9876aaa"), yaml.safe_load(tmp_k8s), ) @@ -207,9 +195,7 @@ def test_datahub_variables(self, mock_git_revision_hash): runner = CliRunner() with tempfile.TemporaryDirectory() as tmp_dir, patch( "data_pipelines_cli.cli_commands.compile.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( - "data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( + ), patch("data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir)), patch( "data_pipelines_cli.cli_constants.BUILD_DIR", pathlib.Path(tmp_dir) ), patch( "data_pipelines_cli.dbt_utils.BUILD_DIR", pathlib.Path(tmp_dir) diff --git a/tests/cli_commands/test_create.py b/tests/cli_commands/test_create.py index b09901a..43823bc 100644 --- a/tests/cli_commands/test_create.py +++ b/tests/cli_commands/test_create.py @@ -36,9 +36,7 @@ def test_create_no_config(self): def test_create_with_template_path(self): with patch("copier.copy", self._mock_copier): runner = CliRunner(mix_stderr=False) - result = runner.invoke( - _cli, ["create", self.copier_dst_path, self.copier_src_path] - ) + result = runner.invoke(_cli, ["create", self.copier_dst_path, self.copier_src_path]) self.assertEqual(0, result.exit_code, msg=result.exception) @patch( @@ -48,9 +46,7 @@ def test_create_with_template_path(self): def test_create_with_template_name(self): with patch("copier.copy", self._mock_copier): runner = CliRunner(mix_stderr=False) - result = runner.invoke( - _cli, ["create", self.copier_dst_path, "create_test"] - ) + result = runner.invoke(_cli, ["create", self.copier_dst_path, "create_test"]) self.assertEqual(0, result.exit_code, msg=result.exception) @patch( diff --git a/tests/cli_commands/test_deploy.py b/tests/cli_commands/test_deploy.py index 582b5a3..62b73d2 100644 --- a/tests/cli_commands/test_deploy.py +++ b/tests/cli_commands/test_deploy.py @@ -53,9 +53,7 @@ def setUp(self) -> None: self.build_temp_dir = pathlib.Path(tempfile.mkdtemp()) dags_path = pathlib.Path(self.build_temp_dir).joinpath("dag") dags_path.mkdir(parents=True) - shutil.copytree( - self.goldens_dir_path.joinpath("config"), dags_path.joinpath("config") - ) + shutil.copytree(self.goldens_dir_path.joinpath("config"), dags_path.joinpath("config")) self.storage_uri = tempfile.mkdtemp() # this way fsspec uses LocalFS @@ -98,9 +96,7 @@ def mock_init( "data_pipelines_cli.cli_commands.deploy.DeployCommand.deploy", lambda _self: _noop, ): - result = runner.invoke( - _cli, ["deploy", "--blob-args", tmp_file.name] - ) + result = runner.invoke(_cli, ["deploy", "--blob-args", tmp_file.name]) self.assertEqual(0, result.exit_code, msg=result.exception) self.assertDictEqual(self.provider_args, result_provider_kwargs) @@ -136,9 +132,7 @@ def test_no_module_cli(self): runner = CliRunner() with patch.dict("sys.modules", **{module_name: None}), patch( "pathlib.Path.cwd", lambda: self.dbt_project_config_dir - ), patch( - "data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir - ): + ), patch("data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir): result = runner.invoke( _cli, [ @@ -158,18 +152,14 @@ def test_no_datahub_method(self): "pathlib.Path.cwd", lambda: self.dbt_project_config_dir ): with self.assertRaises(DependencyNotInstalledError): - DeployCommand( - "base", False, self.storage_uri, self.provider_args, True - ).deploy() + DeployCommand("base", False, self.storage_uri, self.provider_args, True).deploy() @patch("data_pipelines_cli.cli_commands.deploy.BUILD_DIR", goldens_dir_path) def test_datahub_run(self): with patch("pathlib.Path.cwd", lambda: self.dbt_project_config_dir), patch( "data_pipelines_cli.cli_commands.deploy.subprocess_run", self._mock_run ), patch.dict("sys.modules", datahub=MagicMock()): - DeployCommand( - "base", False, self.storage_uri, self.provider_args, True - ).deploy() + DeployCommand("base", False, self.storage_uri, self.provider_args, True).deploy() self.assertListEqual( [ "datahub", @@ -185,9 +175,7 @@ def test_no_docker_method(self): "pathlib.Path.cwd", lambda: self.dbt_project_config_dir ), patch("data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir): with self.assertRaises(DependencyNotInstalledError): - DeployCommand( - "base", True, self.storage_uri, self.provider_args, False - ).deploy() + DeployCommand("base", True, self.storage_uri, self.provider_args, False).deploy() @patch( "data_pipelines_cli.cli_commands.deploy.BUILD_DIR", @@ -261,9 +249,7 @@ def _mock_docker(**kwargs): ), patch( "data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir ): - DeployCommand( - "base", True, self.storage_uri, self.provider_args, False - ).deploy() + DeployCommand("base", True, self.storage_uri, self.provider_args, False).deploy() self.assertEqual("my_docker_repository_uri", docker_kwargs.get("repository")) self.assertEqual("sha1234", docker_kwargs.get("tag")) @@ -287,12 +273,8 @@ def _mock_docker(**_kwargs): with patch("pathlib.Path.cwd", lambda: self.dbt_project_config_dir), patch.dict( "sys.modules", docker=docker_mock - ), patch( - "data_pipelines_cli.data_structures.git_revision_hash", lambda: "sha1234" - ), patch( + ), patch("data_pipelines_cli.data_structures.git_revision_hash", lambda: "sha1234"), patch( "data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir ): with self.assertRaises(DataPipelinesError): - DeployCommand( - "base", True, self.storage_uri, self.provider_args, False - ).deploy() + DeployCommand("base", True, self.storage_uri, self.provider_args, False).deploy() diff --git a/tests/cli_commands/test_prepare_env.py b/tests/cli_commands/test_prepare_env.py index 07592f5..20cc857 100644 --- a/tests/cli_commands/test_prepare_env.py +++ b/tests/cli_commands/test_prepare_env.py @@ -41,10 +41,7 @@ def test_no_var_profiles_generation(self): with tempfile.TemporaryDirectory() as tmp_dir, patch( "data_pipelines_cli.cli_constants.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( - "data_pipelines_cli.config_generation.BUILD_DIR", - pathlib.Path(tmp_dir), - ), patch( + ), patch("data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir),), patch( "data_pipelines_cli.dbt_utils.BUILD_DIR", pathlib.Path(tmp_dir), ), patch( @@ -58,22 +55,15 @@ def test_no_var_profiles_generation(self): with open( pathlib.Path(tmp_dir2).joinpath(".dbt", "profiles.yml"), "r" ) as generated, open( - self.goldens_dir_path.joinpath( - "example_profiles", "local_snowflake.yml" - ), + self.goldens_dir_path.joinpath("example_profiles", "local_snowflake.yml"), "r", ) as prepared: - self.assertDictEqual( - yaml.safe_load(prepared), yaml.safe_load(generated) - ) + self.assertDictEqual(yaml.safe_load(prepared), yaml.safe_load(generated)) def test_vars_profiles_generation(self): with tempfile.TemporaryDirectory() as tmp_dir, patch( "data_pipelines_cli.cli_constants.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( - "data_pipelines_cli.config_generation.BUILD_DIR", - pathlib.Path(tmp_dir), - ), patch( + ), patch("data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir),), patch( "data_pipelines_cli.dbt_utils.BUILD_DIR", pathlib.Path(tmp_dir), ), patch.dict( @@ -87,20 +77,13 @@ def test_vars_profiles_generation(self): ): prepare_env("staging") - with open( - pathlib.Path(tmp_dir2).joinpath(".dbt", "profiles.yml"), "r" - ) as generated: - self.assertDictEqual( - self.rendered_from_vars_profile, yaml.safe_load(generated) - ) + with open(pathlib.Path(tmp_dir2).joinpath(".dbt", "profiles.yml"), "r") as generated: + self.assertDictEqual(self.rendered_from_vars_profile, yaml.safe_load(generated)) def test_raise_missing_variable(self): with tempfile.TemporaryDirectory() as tmp_dir, patch( "data_pipelines_cli.cli_constants.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( - "data_pipelines_cli.config_generation.BUILD_DIR", - pathlib.Path(tmp_dir), - ), patch( + ), patch("data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir),), patch( "data_pipelines_cli.cli_commands.prepare_env.read_dbt_vars_from_configs", lambda _env: {}, ), patch.dict( @@ -118,10 +101,7 @@ def test_raise_missing_variable(self): def test_raise_missing_environment_variable(self): with tempfile.TemporaryDirectory() as tmp_dir, patch( "data_pipelines_cli.cli_constants.BUILD_DIR", pathlib.Path(tmp_dir) - ), patch( - "data_pipelines_cli.config_generation.BUILD_DIR", - pathlib.Path(tmp_dir), - ), patch( + ), patch("data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir),), patch( "data_pipelines_cli.dbt_utils.BUILD_DIR", pathlib.Path(tmp_dir), ), patch.dict( diff --git a/tests/cli_commands/test_publish.py b/tests/cli_commands/test_publish.py index e26ed39..520e2da 100644 --- a/tests/cli_commands/test_publish.py +++ b/tests/cli_commands/test_publish.py @@ -75,9 +75,7 @@ def setUp(self) -> None: self.build_temp_dir = pathlib.Path(tempfile.mkdtemp()) dags_path = pathlib.Path(self.build_temp_dir).joinpath("dag") dags_path.mkdir(parents=True) - shutil.copytree( - goldens_dir_path.joinpath("config"), dags_path.joinpath("config") - ) + shutil.copytree(goldens_dir_path.joinpath("config"), dags_path.joinpath("config")) profiles_yml_path = pathlib.Path(self.build_temp_dir).joinpath( "profiles", "env_execution", "profiles.yml" @@ -111,9 +109,7 @@ def noop(): config_writer_mock = MagicMock() set_value_mock = MagicMock() set_value_mock.configure_mock(**{"release": noop}) - config_writer_mock.configure_mock( - **{"set_value": lambda x, y, z: set_value_mock} - ) + config_writer_mock.configure_mock(**{"set_value": lambda x, y, z: set_value_mock}) repo_mock.configure_mock( **{ "config_writer": config_writer_mock, @@ -132,13 +128,9 @@ def repo_class_mock(self): @patch("pathlib.Path.cwd", lambda: goldens_dir_path) def test_generate_correct_project(self): runner = CliRunner() - with patch( - "data_pipelines_cli.cli_commands.publish.BUILD_DIR", self.build_temp_dir - ), patch( + with patch("data_pipelines_cli.cli_commands.publish.BUILD_DIR", self.build_temp_dir), patch( "data_pipelines_cli.config_generation.BUILD_DIR", self.build_temp_dir - ), patch( - "data_pipelines_cli.cli_commands.publish.Repo", self.repo_class_mock() - ): + ), patch("data_pipelines_cli.cli_commands.publish.Repo", self.repo_class_mock()): runner.invoke(_cli, ["publish", "--key-path", "SOME_KEY.txt"]) result = runner.invoke(_cli, ["publish", "--key-path", "SOME_KEY.txt"]) @@ -158,9 +150,7 @@ def verify_publications(self): def verify_generated_files(self): with open( - pathlib.Path(self.build_temp_dir).joinpath( - "package", "models", "sources.yml" - ), + pathlib.Path(self.build_temp_dir).joinpath("package", "models", "sources.yml"), "r", ) as sources_yml: self.assertDictEqual(self.expected_sources, yaml.safe_load(sources_yml)) @@ -183,9 +173,7 @@ def test_no_models(self): target_path.mkdir(parents=True) with open( goldens_dir_path.joinpath("target", "manifest.json"), "r" - ) as manifest_json, open( - target_path.joinpath("manifest.json"), "w" - ) as tmp_manifest: + ) as manifest_json, open(target_path.joinpath("manifest.json"), "w") as tmp_manifest: manifest = json.load(manifest_json) for k in list(manifest["nodes"].keys()): if k.startswith("model"): diff --git a/tests/goldens/config/base/execution_env.yml b/tests/goldens/config/base/execution_env.yml new file mode 100644 index 0000000..589a23b --- /dev/null +++ b/tests/goldens/config/base/execution_env.yml @@ -0,0 +1,5 @@ +image: + repository: my_docker_repository_uri + tag: + +type: k8s diff --git a/tests/goldens/config/base/k8s.yml b/tests/goldens/config/base/k8s.yml index 04dd1e7..291cc12 100644 --- a/tests/goldens/config/base/k8s.yml +++ b/tests/goldens/config/base/k8s.yml @@ -1,6 +1,3 @@ -image: - repository: my_docker_repository_uri - tag: variable1: 1337 var2: "Hello, world!" envs: diff --git a/tests/test_cli_utils.py b/tests/test_cli_utils.py index 16227f0..42f253b 100644 --- a/tests/test_cli_utils.py +++ b/tests/test_cli_utils.py @@ -36,12 +36,8 @@ def test_echoes_to_proper_streams(self): "sys.stdout", new=StringIO() ) as fake_out, patch("sys.stderr", new=StringIO()) as fake_err: echo_fun(test_string) - self.assertEqual( - endlined_test_string if is_stdout else "", fake_out.getvalue() - ) - self.assertEqual( - endlined_test_string if not is_stdout else "", fake_err.getvalue() - ) + self.assertEqual(endlined_test_string if is_stdout else "", fake_out.getvalue()) + self.assertEqual(endlined_test_string if not is_stdout else "", fake_err.getvalue()) some_env_variable_key = "SOME_VARIABLE" some_env_variable_value = "some_value" @@ -51,26 +47,20 @@ def test_get_argument_from_argument(self): argument = "argument" self.assertEqual( argument, - get_argument_or_environment_variable( - argument, "arg", self.some_env_variable_key - ), + get_argument_or_environment_variable(argument, "arg", self.some_env_variable_key), ) @patch.dict(os.environ, {some_env_variable_key: some_env_variable_value}) def test_get_argument_from_env_var(self): self.assertEqual( self.some_env_variable_value, - get_argument_or_environment_variable( - None, "arg", self.some_env_variable_key - ), + get_argument_or_environment_variable(None, "arg", self.some_env_variable_key), ) @patch.dict(os.environ, {}) def test_get_argument_throw(self): with self.assertRaises(DataPipelinesError): - get_argument_or_environment_variable( - None, "arg", self.some_env_variable_key - ) + get_argument_or_environment_variable(None, "arg", self.some_env_variable_key) @patch("data_pipelines_cli.cli_utils.subprocess.run") def test_subprocess_run_return_code(self, mock_run): diff --git a/tests/test_config_generation.py b/tests/test_config_generation.py index ed9364e..5876726 100644 --- a/tests/test_config_generation.py +++ b/tests/test_config_generation.py @@ -107,18 +107,12 @@ def test_generation(self): ), patch( "pathlib.Path.cwd", lambda: self.goldens_dir_path ): - self.profiles_path = cgen.generate_profiles_yml(env).joinpath( - "profiles.yml" - ) + self.profiles_path = cgen.generate_profiles_yml(env).joinpath("profiles.yml") with open(self.profiles_path, "r") as generated, open( - self.goldens_dir_path.joinpath( - "example_profiles", f"{env}_{profile_type}.yml" - ), + self.goldens_dir_path.joinpath("example_profiles", f"{env}_{profile_type}.yml"), "r", ) as prepared: - self.assertDictEqual( - yaml.safe_load(prepared), yaml.safe_load(generated) - ) + self.assertDictEqual(yaml.safe_load(prepared), yaml.safe_load(generated)) os.remove(self.profiles_path) os.rmdir(self.profiles_path.parent) diff --git a/tests/test_data_structures.py b/tests/test_data_structures.py index 1446b6a..f665bff 100644 --- a/tests/test_data_structures.py +++ b/tests/test_data_structures.py @@ -31,9 +31,7 @@ class DataStructuresTestCase(unittest.TestCase): }, vars={"username": "testuser"}, ) - example_config_path = pathlib.Path(__file__).parent.joinpath( - "goldens", "example_config.yml" - ) + example_config_path = pathlib.Path(__file__).parent.joinpath("goldens", "example_config.yml") def test_read_config(self): with patch( @@ -58,9 +56,7 @@ def setUp(self) -> None: self.build_temp_dir = pathlib.Path(tempfile.mkdtemp()) dags_path = pathlib.Path(self.build_temp_dir).joinpath("dag") dags_path.mkdir(parents=True) - shutil.copytree( - self.goldens_dir_path.joinpath("config"), dags_path.joinpath("config") - ) + shutil.copytree(self.goldens_dir_path.joinpath("config"), dags_path.joinpath("config")) def tearDown(self) -> None: shutil.rmtree(self.build_temp_dir) diff --git a/tests/test_filesystem_utils.py b/tests/test_filesystem_utils.py index e756f24..a87eb16 100644 --- a/tests/test_filesystem_utils.py +++ b/tests/test_filesystem_utils.py @@ -32,9 +32,7 @@ def test_wrong_local_path(self): wrong_local_path = pathlib.Path(__file__).parent.joinpath( "".join(random.choices(string.ascii_letters + string.digits, k=25)) ) - remote_path = "".join( - random.choices(string.ascii_letters + string.digits + "/", k=25) - ) + remote_path = "".join(random.choices(string.ascii_letters + string.digits + "/", k=25)) with self.assertRaises(DataPipelinesError): LocalRemoteSync(wrong_local_path, remote_path, {}).sync(delete=False) @@ -49,9 +47,7 @@ class TestSynchronize(unittest.TestCase): def _test_synchronize(self, protocol: str, **remote_kwargs): from data_pipelines_cli.filesystem_utils import LocalRemoteSync - local_path = pathlib.Path(__file__).parent.joinpath( - "goldens", "test_sync_directory" - ) + local_path = pathlib.Path(__file__).parent.joinpath("goldens", "test_sync_directory") remote_path = f"{protocol}://{MY_BUCKET}/" LocalRemoteSync(local_path, remote_path, remote_kwargs).sync(delete=False) @@ -65,9 +61,7 @@ def _test_synchronize(self, protocol: str, **remote_kwargs): def _test_synchronize_with_delete(self, protocol: str, **remote_kwargs): from data_pipelines_cli.filesystem_utils import LocalRemoteSync - local_path = pathlib.Path(__file__).parent.joinpath( - "goldens", "test_sync_directory" - ) + local_path = pathlib.Path(__file__).parent.joinpath("goldens", "test_sync_directory") remote_path = f"{protocol}://{MY_BUCKET}/" LocalRemoteSync(local_path, remote_path, remote_kwargs).sync(delete=True) @@ -78,9 +72,7 @@ def _test_synchronize_with_delete(self, protocol: str, **remote_kwargs): remote_fs.find(MY_BUCKET), ) - local_path_2 = pathlib.Path(__file__).parent.joinpath( - "goldens", "test_sync_2nd_directory" - ) + local_path_2 = pathlib.Path(__file__).parent.joinpath("goldens", "test_sync_2nd_directory") LocalRemoteSync( local_path_2, remote_path, @@ -151,9 +143,7 @@ class TestGoogleStorageSynchronize(TestSynchronize): def setUp(self) -> None: from gcp_storage_emulator.server import create_server - self.server = create_server( - "localhost", 9023, in_memory=True, default_bucket=MY_BUCKET - ) + self.server = create_server("localhost", 9023, in_memory=True, default_bucket=MY_BUCKET) self.server.start() def tearDown(self): @@ -163,6 +153,4 @@ def test_synchronize(self): self._test_synchronize("gs", endpoint_url="http://localhost:9023", token="anon") def test_synchronize_with_delete(self): - self._test_synchronize_with_delete( - "gs", endpoint_url="http://localhost:9023", token="anon" - ) + self._test_synchronize_with_delete("gs", endpoint_url="http://localhost:9023", token="anon") diff --git a/tests/test_io_utils.py b/tests/test_io_utils.py index d182f48..4c0d8c7 100644 --- a/tests/test_io_utils.py +++ b/tests/test_io_utils.py @@ -49,9 +49,7 @@ def test_git_does_not_exist(self, mock_run): @patch("data_pipelines_cli.io_utils.subprocess.run") def test_git_error(self, mock_run): test_error = "Some test error" - mock_run.side_effect = subprocess.CalledProcessError( - 128, cmd="", stderr=test_error - ) + mock_run.side_effect = subprocess.CalledProcessError(128, cmd="", stderr=test_error) with patch("sys.stderr", new=StringIO()) as fake_out: result = git_revision_hash() diff --git a/tox.ini b/tox.ini index 8c2f186..a91e887 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ commands= # Lint [flake8] exclude = .git,__pycache__,build,dist,docs/source/conf.py -max-line-length = 88 +max-line-length = 100 extend-ignore = E203 [mypy]