Skip to content

Commit

Permalink
[FIX] unused SparkSession being import from pyspark.sql in several te…
Browse files Browse the repository at this point in the history
…sts (#140)

<!--- Provide a general summary of your changes in the Title above -->

## Description
<!--- Describe your changes in detail -->

## Related Issue
<!--- This project only accepts pull requests related to open issues -->
<!--- If suggesting a new feature or change, please discuss it in an
issue first -->
<!--- If fixing a bug, there should be an issue describing it with steps
to reproduce -->
<!--- Please link to the issue here: -->
#139

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->

## How Has This Been Tested?
<!--- Please describe in detail how you tested your changes. -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->

## Screenshots (if appropriate):

## Types of changes
<!--- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->
- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)

## Checklist:
<!--- Go over all the following points, and put an `x` in all the boxes
that apply. -->
<!--- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
- [x] My code follows the code style of this project.
- [ ] My change requires a change to the documentation.
- [ ] I have updated the documentation accordingly.
- [x] I have read the **CONTRIBUTING** document.
- [x] I have added tests to cover my changes.
- [x] All new and existing tests passed.

Co-authored-by: Danny Meijer <[email protected]>
  • Loading branch information
dannymeijer and dannymeijer authored Nov 29, 2024
1 parent 1e21e37 commit 5298c2b
Show file tree
Hide file tree
Showing 5 changed files with 5 additions and 17 deletions.
1 change: 0 additions & 1 deletion tests/spark/integrations/dq/test_spark_expectations.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import pytest

import pyspark
from pyspark.sql import SparkSession

from koheesio.utils import get_project_root
Expand Down
4 changes: 0 additions & 4 deletions tests/spark/readers/test_hana.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
from unittest import mock

import pytest

from pyspark.sql import SparkSession

from koheesio.spark.readers.hana import HanaReader

pytestmark = pytest.mark.spark
Expand Down
4 changes: 0 additions & 4 deletions tests/spark/readers/test_jdbc.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
from unittest import mock

import pytest

from pyspark.sql import SparkSession

from koheesio.spark.readers.jdbc import JdbcReader

pytestmark = pytest.mark.spark
Expand Down
4 changes: 1 addition & 3 deletions tests/spark/transformations/test_lookup.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import pytest

from pyspark.sql import SparkSession

from koheesio.spark.transformations.lookup import (
DataframeLookup,
JoinHint,
Expand Down Expand Up @@ -37,7 +35,7 @@ def test_join_hint_values() -> None:


@pytest.mark.parametrize("join_hint", [None, JoinHint.BROADCAST])
def test_dataframe_lookup(spark: SparkSession, join_hint: JoinHint) -> None:
def test_dataframe_lookup(spark, join_hint: JoinHint) -> None:
df = spark.createDataFrame(
[("1", "a", "a"), ("2", "b", "b")],
schema="key string, second_key string, field string",
Expand Down
9 changes: 4 additions & 5 deletions tests/spark/transformations/test_row_number_dedup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import pytest

from pyspark.sql import SparkSession
from pyspark.sql import functions as F

from koheesio.spark.transformations.row_number_dedup import RowNumberDedup
Expand All @@ -11,7 +10,7 @@


@pytest.mark.parametrize("target_column", ["col_row_number"])
def test_row_number_dedup(spark: SparkSession, target_column: str) -> None:
def test_row_number_dedup(spark, target_column: str) -> None:
df = spark.createDataFrame(
[
(
Expand Down Expand Up @@ -49,7 +48,7 @@ def test_row_number_dedup(spark: SparkSession, target_column: str) -> None:


@pytest.mark.parametrize("target_column", ["col_row_number"])
def test_row_number_dedup_not_list_column(spark: SparkSession, target_column: str) -> None:
def test_row_number_dedup_not_list_column(spark, target_column: str) -> None:
df = spark.createDataFrame(
[
(
Expand Down Expand Up @@ -89,7 +88,7 @@ def test_row_number_dedup_not_list_column(spark: SparkSession, target_column: st


@pytest.mark.parametrize("target_column", ["col_row_number"])
def test_row_number_dedup_with_columns(spark: SparkSession, target_column: str) -> None:
def test_row_number_dedup_with_columns(spark, target_column: str) -> None:
df = spark.createDataFrame(
[
(
Expand Down Expand Up @@ -129,7 +128,7 @@ def test_row_number_dedup_with_columns(spark: SparkSession, target_column: str)


@pytest.mark.parametrize("target_column", ["col_row_number"])
def test_row_number_dedup_with_duplicated_columns(spark: SparkSession, target_column: str) -> None:
def test_row_number_dedup_with_duplicated_columns(spark, target_column: str) -> None:
df = spark.createDataFrame(
[
(
Expand Down

0 comments on commit 5298c2b

Please sign in to comment.