diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9298c7d..11eb9d7e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,8 +83,9 @@ jobs: with: name: python-artifacts path: dist + + - name: Install Hatch + uses: pypa/hatch@install - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.14 - with: - print-hash: true \ No newline at end of file + run: hatch publish --yes --no-prompt \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index bb902803..b834861a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -289,12 +289,26 @@ matrix.version.extra-dependencies = [ "pyspark35", ] }, ] + +name.".*pyspark35$".features = [ + "async", + "async_http", + "box", + "pandas", + "pyspark", + "sftp", + "delta", + "dev", + "test", +] + name.".*".env-vars = [ # set number of workes for parallel testing { key = "PYTEST_XDIST_AUTO_NUM_WORKERS", value = "2" }, # disables Koheesio logo being printed during testing { key = "KOHEESIO__PRINT_LOGO", value = "False" }, ] + [tool.pytest.ini_options] addopts = "-q --color=yes --order-scope=module" log_level = "CRITICAL" diff --git a/tests/spark/integrations/dq/test_spark_expectations.py b/tests/spark/integrations/dq/test_spark_expectations.py index 94b631d7..a8ef6fbf 100644 --- a/tests/spark/integrations/dq/test_spark_expectations.py +++ b/tests/spark/integrations/dq/test_spark_expectations.py @@ -1,15 +1,17 @@ from typing import List, Union +import pyspark import pytest - -from pyspark.sql import SparkSession - from koheesio.utils import get_project_root +from pyspark.sql import SparkSession PROJECT_ROOT = get_project_root() pytestmark = pytest.mark.spark +if pyspark.__version__.startswith("3.5"): + pytestmark = pytest.mark.skip("Spark Expectations is not supported for Spark 3.5") + class TestSparkExpectationsTransform: """