Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactoring/#217 rename dataflow abstraction files #218

Merged
merged 4 commits into from
Nov 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/changes/changes_0.1.1.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,4 @@ Code name:
* #188: Renamed global pytest fixtures to avoid name clashes
* #208: Replaced access to private attribute by public
* #203: Cleaned-up package names and directory structure
* #217: Rename dataflow abstraction files
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from exasol.analytics.utils.data_classes_runtime_type_check import check_dataclass_types

from exasol.analytics.query_handler.graph.stage.sql.execution.dependency import Dependencies
from exasol.analytics.query_handler.graph.stage.sql.dependency import Dependencies


@dataclasses.dataclass(frozen=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from exasol.analytics.utils.data_classes_runtime_type_check import check_dataclass_types

from exasol.analytics.query_handler.graph.stage.sql.execution.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.data_partition import DataPartition

DataPartitionName = Union[Enum, Tuple[Enum, int]]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from exasol_bucketfs_utils_python.abstract_bucketfs_location import AbstractBucketFSLocation

from exasol.analytics.query_handler.graph.stage.sql.sql_stage_graph import SQLStageGraph
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput


@dataclasses.dataclass(frozen=True, eq=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from exasol.analytics.query_handler.graph.stage.sql.execution.input import SQLStageGraphExecutionInput
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler_state import SQLStageGraphExecutionQueryHandlerState, ResultHandlerReturnValue
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput

SQLStageGraphExecutionQueryHandlerStateFactory = \
Callable[
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
from exasol.analytics.query_handler.graph.stage.sql.execution.find_object_proxies import find_object_proxies
from exasol.analytics.query_handler.graph.stage.sql.execution.object_proxy_reference_counting_bag import ObjectProxyReferenceCountingBag
from exasol.analytics.query_handler.graph.stage.sql.execution.input import SQLStageGraphExecutionInput
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_train_query_handler import SQLStageTrainQueryHandlerInput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_query_handler import SQLStageTrainQueryHandlerInput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage import SQLStage


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

from exasol.analytics.utils.data_classes_runtime_type_check import check_dataclass_types

from exasol.analytics.query_handler.graph.stage.sql.execution.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.execution.dependency import Dependencies
from exasol.analytics.query_handler.graph.stage.sql.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.dependency import Dependencies


@dataclasses.dataclass(frozen=True, eq=True)
Expand Down
4 changes: 2 additions & 2 deletions exasol/analytics/query_handler/graph/stage/sql/sql_stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

from exasol.analytics.query_handler.context.scope import ScopeQueryHandlerContext
from exasol.analytics.query_handler.graph.stage.stage import Stage
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_train_query_handler import SQLStageTrainQueryHandler, SQLStageTrainQueryHandlerInput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_query_handler import SQLStageQueryHandler, SQLStageTrainQueryHandlerInput

class SQLStage(Stage):
@abc.abstractmethod
def create_train_query_handler(
self,
stage_input: SQLStageTrainQueryHandlerInput,
query_handler_context: ScopeQueryHandlerContext,
) -> SQLStageTrainQueryHandler:
) -> SQLStageQueryHandler:
pass
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from exasol.analytics.query_handler.execution_graph import ExecutionGraph
from exasol.analytics.query_handler.graph.execution_graph import ExecutionGraph
from exasol.analytics.query_handler.graph.stage.sql.sql_stage import SQLStage

SQLStageGraph = ExecutionGraph[SQLStage]
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from exasol.analytics.query_handler.query_handler import QueryHandler
from exasol_bucketfs_utils_python.abstract_bucketfs_location import AbstractBucketFSLocation

from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput


def is_empty(obj: Sized):
Expand All @@ -22,6 +22,6 @@ def __post_init__(self):
raise AssertionError("Empty sql_stage_inputs not allowed.")


class SQLStageTrainQueryHandler(
class SQLStageQueryHandler(
QueryHandler[SQLStageTrainQueryHandlerInput, SQLStageInputOutput], ABC):
pass
2 changes: 1 addition & 1 deletion exasol/analytics/query_handler/graph/stage/stage_graph.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from exasol.analytics.query_handler.execution_graph import ExecutionGraph
from exasol.analytics.query_handler.graph.execution_graph import ExecutionGraph
from exasol.analytics.query_handler.graph.stage.stage import Stage

StageGraph = ExecutionGraph[Stage]
2 changes: 1 addition & 1 deletion noxconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
class Config:
root: Path = ROOT_DIR
doc: Path = ROOT_DIR / "doc"
# version_file: Path = ROOT_DIR / "version.py"
version_file: Path = ROOT_DIR / "version.py"
path_filters: Iterable[str] = ("dist", ".eggs", "venv")


Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from typing import List

from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_train_query_handler import SQLStageTrainQueryHandlerInput
from tests.unit_tests.sql_stage_graph.mock_cast import mock_cast
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_query_handler import SQLStageTrainQueryHandlerInput
from tests.mock_cast import mock_cast
from tests.unit_tests.sql_stage_graph.stage_graph_execution_query_handler.state_test_setup import TestSetup


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@
from exasol.analytics.query_handler.graph.stage.sql.execution.object_proxy_reference_counting_bag import ObjectProxyReferenceCountingBag
from exasol.analytics.query_handler.graph.stage.sql.execution.input import SQLStageGraphExecutionInput
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler_state import SQLStageGraphExecutionQueryHandlerState
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage import SQLStage
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_train_query_handler import SQLStageTrainQueryHandler
from tests.unit_tests.sql_stage_graph.mock_cast import mock_cast
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_query_handler import SQLStageQueryHandler
from tests.mock_cast import mock_cast

MockScopeQueryHandlerContext = Union[ScopeQueryHandlerContext, MagicMock]
MockSQLStageTrainQueryHandler = Union[SQLStageTrainQueryHandler, MagicMock]
MockSQLStageTrainQueryHandler = Union[SQLStageQueryHandler, MagicMock]
MockQueryHandlerResult = Union[Continue, Finish, MagicMock]
MockSQLStageInputOutput = Union[SQLStageInputOutput, MagicMock]
MockSQLStage = Union[SQLStage, MagicMock]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@
ColumnNameBuilder,
)
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_graph import SQLStageGraph
from exasol.analytics.query_handler.graph.stage.sql.execution.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.execution.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.execution.input import SQLStageGraphExecutionInput
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler import SQLStageGraphExecutionQueryHandler
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_train_query_handler import SQLStageTrainQueryHandler, SQLStageTrainQueryHandlerInput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_query_handler import SQLStageQueryHandler, SQLStageTrainQueryHandlerInput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage import SQLStage


class StartOnlyForwardInputTestSQLStageTrainQueryHandler(SQLStageTrainQueryHandler):
class StartOnlyForwardInputTestSQLStageTrainQueryHandler(SQLStageQueryHandler):

def __init__(self, parameter: SQLStageTrainQueryHandlerInput,
query_handler_context: ScopeQueryHandlerContext):
Expand All @@ -47,7 +47,7 @@ def handle_query_result(self, query_result: QueryResult) \
raise NotImplementedError()


class StartOnlyCreateNewOutputTestSQLStageTrainQueryHandler(SQLStageTrainQueryHandler):
class StartOnlyCreateNewOutputTestSQLStageTrainQueryHandler(SQLStageQueryHandler):

def __init__(self, parameter: SQLStageTrainQueryHandlerInput,
query_handler_context: ScopeQueryHandlerContext):
Expand All @@ -71,7 +71,7 @@ def handle_query_result(self, query_result: QueryResult) \
raise NotImplementedError()


class HandleQueryResultCreateNewOutputTestSQLStageTrainQueryHandler(SQLStageTrainQueryHandler):
class HandleQueryResultCreateNewOutputTestSQLStageTrainQueryHandler(SQLStageQueryHandler):

def __init__(self, parameter: SQLStageTrainQueryHandlerInput,
query_handler_context: ScopeQueryHandlerContext):
Expand Down Expand Up @@ -102,7 +102,7 @@ def handle_query_result(self, query_result: QueryResult) \


TrainQueryHandlerFactory = Callable[
[SQLStageTrainQueryHandlerInput, ScopeQueryHandlerContext], SQLStageTrainQueryHandler]
[SQLStageTrainQueryHandlerInput, ScopeQueryHandlerContext], SQLStageQueryHandler]


class TestSQLStage(SQLStage):
Expand All @@ -112,11 +112,11 @@ def __init__(self, *,
index: int,
train_query_handler_factory: TrainQueryHandlerFactory):
self._train_query_handler_factory = train_query_handler_factory
self.sql_stage_train_query_handler: Optional[SQLStageTrainQueryHandler] = None
self.sql_stage_train_query_handler: Optional[SQLStageQueryHandler] = None
self._index = index

def create_train_query_handler(self, query_handler_input: SQLStageTrainQueryHandlerInput,
query_handler_context: ScopeQueryHandlerContext) -> SQLStageTrainQueryHandler:
query_handler_context: ScopeQueryHandlerContext) -> SQLStageQueryHandler:
self.sql_stage_train_query_handler = self._train_query_handler_factory(query_handler_input,
query_handler_context)
return self.sql_stage_train_query_handler
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@
from exasol.analytics.query_handler.graph.stage.sql.execution.input import SQLStageGraphExecutionInput
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler import SQLStageGraphExecutionQueryHandler, SQLStageGraphExecutionQueryHandlerStateFactory
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler_state import SQLStageGraphExecutionQueryHandlerState, ResultHandlerReturnValue
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_train_query_handler import SQLStageTrainQueryHandler
from tests.unit_tests.sql_stage_graph.mock_cast import mock_cast
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_query_handler import SQLStageQueryHandler
from tests.mock_cast import mock_cast

MockSQLStageGraphExecutionQueryHandlerState = Union[SQLStageGraphExecutionQueryHandlerState, MagicMock]
MockScopeQueryHandlerContext = Union[ScopeQueryHandlerContext, MagicMock]
MockSQLStageTrainQueryHandler = Union[SQLStageTrainQueryHandler, MagicMock]
MockSQLStageTrainQueryHandler = Union[SQLStageQueryHandler, MagicMock]
MockQueryHandlerResult = Union[Continue, Finish, MagicMock]
MockSQLStageGraphExecutionInput = Union[SQLStageGraphExecutionInput, MagicMock]
MockSQLStageInputOutput = Union[SQLStageInputOutput, MagicMock]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from exasol.analytics.query_handler.graph.stage.sql.sql_stage_graph import SQLStageGraph
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler_state import ResultHandlerReturnValue
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from tests.unit_tests.sql_stage_graph.stage_graph_execution_query_handler.assert_helper import \
assert_reference_counting_bag_creation, assert_stage_train_query_handler_created, assert_stage_not_called, \
assert_reference_counting_bag_not_called, assert_release_on_query_handler_context_for_stage, \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from exasol.analytics.query_handler.graph.stage.sql.sql_stage_graph import SQLStageGraph
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler_state import ResultHandlerReturnValue
from tests.unit_tests.sql_stage_graph.mock_cast import mock_cast
from tests.mock_cast import mock_cast
from tests.unit_tests.sql_stage_graph.stage_graph_execution_query_handler.state_test_setup import TestSetup, \
create_mocks_for_stage, create_execution_query_handler_state_setup

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from exasol.analytics.query_handler.graph.stage.sql.sql_stage_graph import SQLStageGraph
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler_state import ResultHandlerReturnValue
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from tests.unit_tests.sql_stage_graph.stage_graph_execution_query_handler.assert_helper import assert_stage_not_called, \
assert_reference_counting_bag_not_called, assert_reference_counting_bag_creation, \
assert_stage_train_query_handler_created, \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from exasol.analytics.query_handler.graph.stage.sql.sql_stage_graph import SQLStageGraph
from exasol.analytics.query_handler.graph.stage.sql.execution.query_handler_state import ResultHandlerReturnValue
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from tests.unit_tests.sql_stage_graph.stage_graph_execution_query_handler.assert_helper import \
assert_reference_counting_bag_creation, \
assert_stage_train_query_handler_created, assert_stage_not_called, \
Expand Down
4 changes: 2 additions & 2 deletions tests/unit_tests/sql_stage_graph/test_data_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
ColumnType,
)

from exasol.analytics.query_handler.graph.stage.sql.execution.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.execution.dependency import Dependency
from exasol.analytics.query_handler.graph.stage.sql.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.dependency import Dependency


class TestEnum(Enum):
Expand Down
6 changes: 3 additions & 3 deletions tests/unit_tests/sql_stage_graph/test_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@
ColumnType,
)

from exasol.analytics.query_handler.graph.stage.sql.execution.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.execution.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.execution.dependency import Dependencies
from exasol.analytics.query_handler.graph.stage.sql.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.dependency import Dependencies


class TestEnum(Enum):
Expand Down
2 changes: 1 addition & 1 deletion tests/unit_tests/sql_stage_graph/test_dependency.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from enum import Enum, auto

from exasol.analytics.query_handler.graph.stage.sql.execution.dependency import Dependency
from exasol.analytics.query_handler.graph.stage.sql.dependency import Dependency


class TestEnum(Enum):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
ColumnType,
ColumnNameBuilder,
)
from exasol.analytics.query_handler.graph.stage.sql.execution.dependency import Dependency
from exasol.analytics.query_handler.graph.stage.sql.dependency import Dependency
from exasol.analytics.query_handler.graph.stage.sql.execution.find_object_proxies import find_object_proxies

BUCKETFS_LOCATION = "BUCKETFS_LOCATION"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from exasol.analytics.query_handler.context.scope import ScopeQueryHandlerContext

from exasol.analytics.query_handler.graph.stage.sql.execution.object_proxy_reference_counter import ObjectProxyReferenceCounter, ReferenceCounterStatus
from tests.unit_tests.sql_stage_graph.mock_cast import mock_cast
from tests.mock_cast import mock_cast

MockScopeQueryHandlerContext = Union[ScopeQueryHandlerContext, MagicMock]
MockObjectProxy = Union[ObjectProxy, MagicMock]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from exasol.analytics.query_handler.graph.stage.sql.execution.object_proxy_reference_counter import ObjectProxyReferenceCounter, ReferenceCounterStatus
from exasol.analytics.query_handler.graph.stage.sql.execution.object_proxy_reference_counting_bag import ObjectProxyReferenceCountingBag, ObjectProxyReferenceCounterFactory
from tests.unit_tests.sql_stage_graph.mock_cast import mock_cast
from tests.mock_cast import mock_cast

MockObjectProxyReferenceCounter = Union[ObjectProxyReferenceCounter, MagicMock]
MockObjectProxyReferenceCounterFactory = Union[ObjectProxyReferenceCounterFactory, Mock]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
TableNameBuilder,
ColumnType,
)
from exasol.analytics.query_handler.graph.stage.sql.execution.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.execution.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.execution.dependency import Dependencies, Dependency
from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.data_partition import DataPartition
from exasol.analytics.query_handler.graph.stage.sql.dataset import Dataset
from exasol.analytics.query_handler.graph.stage.sql.dependency import Dependencies, Dependency
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput


class TestEnum(Enum):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import pytest
from exasol_bucketfs_utils_python.abstract_bucketfs_location import AbstractBucketFSLocation

from exasol.analytics.query_handler.graph.stage.sql.execution.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_train_query_handler import SQLStageTrainQueryHandlerInput
from exasol.analytics.query_handler.graph.stage.sql.input_output import SQLStageInputOutput
from exasol.analytics.query_handler.graph.stage.sql.sql_stage_query_handler import SQLStageTrainQueryHandlerInput


def test_empty_stage_inputs():
Expand Down
10 changes: 10 additions & 0 deletions version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# ATTENTION:
# This file is generated by exasol/toolbox/pre_commit_hooks/package_version.py when using:
# * either "poetry run nox -s fix"
# * or "poetry run version-check <path/version.py> --fix"
# Do not edit this file manually!
# If you need to change the version, do so in the project.toml, e.g. by using `poetry version X.Y.Z`.
MAJOR = 0
MINOR = 1
PATCH = 0
VERSION = f"{MAJOR}.{MINOR}.{PATCH}"