diff --git a/datajunction-server/datajunction_server/alembic/env.py b/datajunction-server/datajunction_server/alembic/env.py index 0d0f6d7f2..1e28f0bf0 100644 --- a/datajunction-server/datajunction_server/alembic/env.py +++ b/datajunction-server/datajunction_server/alembic/env.py @@ -1,7 +1,6 @@ """ Environment for Alembic migrations. """ -# pylint: disable=no-member, unused-import, no-name-in-module, import-error import os from logging.config import fileConfig diff --git a/datajunction-server/datajunction_server/api/access/authentication/google.py b/datajunction-server/datajunction_server/api/access/authentication/google.py index de7262b7e..e94381f3e 100644 --- a/datajunction-server/datajunction_server/api/access/authentication/google.py +++ b/datajunction-server/datajunction_server/api/access/authentication/google.py @@ -70,7 +70,7 @@ async def get_access_token( request_session = requests.session() token_request = google.auth.transport.requests.Request(session=request_session) user_data = id_token.verify_oauth2_token( - id_token=credentials._id_token, # pylint: disable=protected-access + id_token=credentials._id_token, request=token_request, audience=setting.google_oauth_client_id, ) diff --git a/datajunction-server/datajunction_server/api/attributes.py b/datajunction-server/datajunction_server/api/attributes.py index 88a2c854f..7cd1e5a6d 100644 --- a/datajunction-server/datajunction_server/api/attributes.py +++ b/datajunction-server/datajunction_server/api/attributes.py @@ -101,7 +101,6 @@ async def default_attribute_types(session: AsyncSession = Depends(get_session)): # Update existing default attribute types statement = select(AttributeType).filter( - # pylint: disable=no-member AttributeType.name.in_( # type: ignore set(default_attribute_type_names.keys()), ), diff --git a/datajunction-server/datajunction_server/api/cubes.py b/datajunction-server/datajunction_server/api/cubes.py index 5556f2352..0a493a98b 100644 --- a/datajunction-server/datajunction_server/api/cubes.py +++ b/datajunction-server/datajunction_server/api/cubes.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-arguments """ Cube related APIs. """ @@ -164,7 +163,7 @@ async def get_cube_dimension_sql( include_counts: bool = False, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=redefined-outer-name + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> TranslatedSQL: @@ -189,7 +188,7 @@ async def get_cube_dimension_sql( "/cubes/{name}/dimensions/data", name="Dimensions Values for Cube", ) -async def get_cube_dimension_values( # pylint: disable=too-many-locals +async def get_cube_dimension_values( name: str, *, dimensions: List[str] = Query([], description="Dimensions to get values for"), @@ -207,7 +206,7 @@ async def get_cube_dimension_values( # pylint: disable=too-many-locals request: Request, query_service_client: QueryServiceClient = Depends(get_query_service_client), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=redefined-outer-name + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> DimensionValues: @@ -260,7 +259,7 @@ async def get_cube_dimension_values( # pylint: disable=too-many-locals return DimensionValues( # pragma: no cover dimensions=[ from_amenable_name(col.name) - for col in translated_sql.columns # type: ignore # pylint: disable=not-an-iterable + for col in translated_sql.columns # type: ignore if col.name != "count" ], values=dimension_values, diff --git a/datajunction-server/datajunction_server/api/data.py b/datajunction-server/datajunction_server/api/data.py index 69a2f04e8..a69d09796 100644 --- a/datajunction-server/datajunction_server/api/data.py +++ b/datajunction-server/datajunction_server/api/data.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-arguments """ Data related APIs. """ @@ -57,7 +56,7 @@ async def add_availability_state( *, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), notify: Callable = Depends(get_notifier), @@ -164,7 +163,7 @@ async def add_availability_state( @router.get("/data/{node_name}/", name="Get Data for a Node") -async def get_data( # pylint: disable=too-many-locals +async def get_data( node_name: str, *, dimensions: List[str] = Query([], description="Dimensional attributes to group by"), @@ -184,7 +183,7 @@ async def get_data( # pylint: disable=too-many-locals engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), background_tasks: BackgroundTasks, @@ -243,7 +242,7 @@ async def get_data( # pylint: disable=too-many-locals @router.get("/stream/{node_name}", response_model=QueryWithResults) -async def get_data_stream_for_node( # pylint: disable=R0914, R0913 +async def get_data_stream_for_node( node_name: str, *, dimensions: List[str] = Query([], description="Dimensional attributes to group by"), @@ -259,7 +258,7 @@ async def get_data_stream_for_node( # pylint: disable=R0914, R0913 engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), background_tasks: BackgroundTasks, @@ -365,7 +364,7 @@ def get_data_for_query( @router.get("/data/", response_model=QueryWithResults, name="Get Data For Metrics") -async def get_data_for_metrics( # pylint: disable=R0914, R0913 +async def get_data_for_metrics( metrics: List[str] = Query([]), dimensions: List[str] = Query([]), filters: List[str] = Query([]), @@ -379,7 +378,7 @@ async def get_data_for_metrics( # pylint: disable=R0914, R0913 engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> QueryWithResults: @@ -424,7 +423,7 @@ async def get_data_for_metrics( # pylint: disable=R0914, R0913 @router.get("/stream/", response_model=QueryWithResults) -async def get_data_stream_for_metrics( # pylint: disable=R0914, R0913 +async def get_data_stream_for_metrics( metrics: List[str] = Query([]), dimensions: List[str] = Query([]), filters: List[str] = Query([]), @@ -437,7 +436,7 @@ async def get_data_stream_for_metrics( # pylint: disable=R0914, R0913 engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> QueryWithResults: diff --git a/datajunction-server/datajunction_server/api/dimensions.py b/datajunction-server/datajunction_server/api/dimensions.py index d6d85c713..3e6d3fabf 100644 --- a/datajunction-server/datajunction_server/api/dimensions.py +++ b/datajunction-server/datajunction_server/api/dimensions.py @@ -42,7 +42,7 @@ async def list_dimensions( *, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> List[NodeIndegreeOutput]: @@ -73,7 +73,7 @@ async def find_nodes_with_dimension( node_type: List[NodeType] = Query([]), session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> List[NodeRevisionOutput]: @@ -106,7 +106,7 @@ async def find_nodes_with_common_dimensions( *, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> List[NodeRevisionOutput]: diff --git a/datajunction-server/datajunction_server/api/djsql.py b/datajunction-server/datajunction_server/api/djsql.py index 7a02a68e1..795d8ff71 100644 --- a/datajunction-server/datajunction_server/api/djsql.py +++ b/datajunction-server/datajunction_server/api/djsql.py @@ -27,7 +27,7 @@ @router.get("/djsql/data", response_model=QueryWithResults) -async def get_data_for_djsql( # pylint: disable=R0914, R0913 +async def get_data_for_djsql( query: str, async_: bool = False, *, @@ -37,7 +37,7 @@ async def get_data_for_djsql( # pylint: disable=R0914, R0913 engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> QueryWithResults: @@ -77,7 +77,6 @@ async def get_data_for_djsql( # pylint: disable=R0914, R0913 return result -# pylint: disable=R0914, R0913 @router.get("/djsql/stream/", response_model=QueryWithResults) async def get_data_stream_for_djsql( query: str, @@ -88,7 +87,7 @@ async def get_data_stream_for_djsql( engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> QueryWithResults: # pragma: no cover diff --git a/datajunction-server/datajunction_server/api/graphql/main.py b/datajunction-server/datajunction_server/api/graphql/main.py index 32715b254..df6fab095 100644 --- a/datajunction-server/datajunction_server/api/graphql/main.py +++ b/datajunction-server/datajunction_server/api/graphql/main.py @@ -1,8 +1,12 @@ """DJ graphql""" +import logging +from functools import wraps + import strawberry from fastapi import Depends from strawberry.fastapi import GraphQLRouter +from strawberry.types import Info from datajunction_server.api.graphql.queries.catalogs import list_catalogs from datajunction_server.api.graphql.queries.dag import common_dimensions @@ -20,6 +24,43 @@ from datajunction_server.api.graphql.scalars.tag import Tag from datajunction_server.utils import get_session, get_settings +logger = logging.getLogger(__name__) + + +def log_resolver(func): + """ + Adds generic logging to the GQL resolver. + """ + + @wraps(func) + async def wrapper(*args, **kwargs): + resolver_name = func.__name__ + + info: Info = kwargs.get("info") if "info" in kwargs else None + user = info.context.get("user", "anonymous") if info else "unknown" + args_dict = {key: val for key, val in kwargs.items() if key != "info"} + log_tags = { + "query_name": resolver_name, + "user": user, + **args_dict, + } + log_args = " ".join( + [f"{tag}={value}" for tag, value in log_tags.items() if value], + ) + try: + result = await func(*args, **kwargs) + logger.info("[GQL] %s", log_args) + return result + except Exception as exc: # pragma: no cover + logger.error( # pragma: no cover + "[GQL] status=error %s", + log_args, + exc_info=True, + ) + raise exc # pragma: no cover + + return wrapper + async def get_context( session=Depends(get_session), @@ -32,47 +73,47 @@ async def get_context( @strawberry.type -class Query: # pylint: disable=R0903 +class Query: """ Parent of all DJ graphql queries """ # Catalog and engine queries - list_catalogs: list[Catalog] = strawberry.field( # noqa: F811 - resolver=list_catalogs, + list_catalogs: list[Catalog] = strawberry.field( + resolver=log_resolver(list_catalogs), ) - list_engines: list[Engine] = strawberry.field( # noqa: F811 - resolver=list_engines, + list_engines: list[Engine] = strawberry.field( + resolver=log_resolver(list_engines), ) # Node search queries - find_nodes: list[Node] = strawberry.field( # noqa: F811 - resolver=find_nodes, + find_nodes: list[Node] = strawberry.field( + resolver=log_resolver(find_nodes), description="Find nodes based on the search parameters.", ) - find_nodes_paginated: Connection[Node] = strawberry.field( # noqa: F811 - resolver=find_nodes_paginated, + find_nodes_paginated: Connection[Node] = strawberry.field( + resolver=log_resolver(find_nodes_paginated), description="Find nodes based on the search parameters with pagination", ) # DAG queries - common_dimensions: list[DimensionAttribute] = strawberry.field( # noqa: F811 - resolver=common_dimensions, + common_dimensions: list[DimensionAttribute] = strawberry.field( + resolver=log_resolver(common_dimensions), description="Get common dimensions for one or more nodes", ) # Generate SQL queries - measures_sql: list[GeneratedSQL] = strawberry.field( # noqa: F811 - resolver=measures_sql, + measures_sql: list[GeneratedSQL] = strawberry.field( + resolver=log_resolver(measures_sql), ) # Tags queries - list_tags: list[Tag] = strawberry.field( # noqa: F811 - resolver=list_tags, + list_tags: list[Tag] = strawberry.field( + resolver=log_resolver(list_tags), description="Find DJ node tags based on the search parameters.", ) - list_tag_types: list[str] = strawberry.field( # noqa: F811 - resolver=list_tag_types, + list_tag_types: list[str] = strawberry.field( + resolver=log_resolver(list_tag_types), description="List all DJ node tag types", ) diff --git a/datajunction-server/datajunction_server/api/graphql/queries/catalogs.py b/datajunction-server/datajunction_server/api/graphql/queries/catalogs.py index 008b83042..65e49fb04 100644 --- a/datajunction-server/datajunction_server/api/graphql/queries/catalogs.py +++ b/datajunction-server/datajunction_server/api/graphql/queries/catalogs.py @@ -7,9 +7,7 @@ from sqlalchemy import select from strawberry.types import Info -from datajunction_server.api.graphql.scalars.catalog_engine import ( - Catalog, # pylint: disable=W0611 -) +from datajunction_server.api.graphql.scalars.catalog_engine import Catalog from datajunction_server.database.catalog import Catalog as DBCatalog @@ -22,6 +20,6 @@ async def list_catalogs( """ session = info.context["session"] # type: ignore return [ - Catalog.from_pydantic(catalog) # type: ignore #pylint: disable=E1101 + Catalog.from_pydantic(catalog) # type: ignore for catalog in (await session.execute(select(DBCatalog))).scalars().all() ] diff --git a/datajunction-server/datajunction_server/api/graphql/queries/sql.py b/datajunction-server/datajunction_server/api/graphql/queries/sql.py index 36a350107..b7306f32a 100644 --- a/datajunction-server/datajunction_server/api/graphql/queries/sql.py +++ b/datajunction-server/datajunction_server/api/graphql/queries/sql.py @@ -10,7 +10,7 @@ @strawberry.input -class CubeDefinition: # pylint: disable=too-few-public-methods +class CubeDefinition: """ The cube definition for the query """ @@ -42,7 +42,7 @@ class CubeDefinition: # pylint: disable=too-few-public-methods @strawberry.input -class EngineSettings: # pylint: disable=too-few-public-methods +class EngineSettings: """ The engine settings for the query """ diff --git a/datajunction-server/datajunction_server/api/graphql/resolvers/nodes.py b/datajunction-server/datajunction_server/api/graphql/resolvers/nodes.py index e5e1e7dfb..c653ffa4f 100644 --- a/datajunction-server/datajunction_server/api/graphql/resolvers/nodes.py +++ b/datajunction-server/datajunction_server/api/graphql/resolvers/nodes.py @@ -53,7 +53,7 @@ async def find_nodes_by( limit, before, after, - *options, + options=options, ) diff --git a/datajunction-server/datajunction_server/api/graphql/resolvers/tags.py b/datajunction-server/datajunction_server/api/graphql/resolvers/tags.py index 691cd419c..8a93517d2 100644 --- a/datajunction-server/datajunction_server/api/graphql/resolvers/tags.py +++ b/datajunction-server/datajunction_server/api/graphql/resolvers/tags.py @@ -19,9 +19,7 @@ async def get_nodes_by_tag( Retrieves all nodes with the given tag. A list of fields must be requested on the node, or this will not return any data. """ - from datajunction_server.api.graphql.resolvers.nodes import ( # pylint: disable=import-outside-toplevel - load_node_options, - ) + from datajunction_server.api.graphql.resolvers.nodes import load_node_options options = load_node_options( fields["nodes"] diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/__init__.py b/datajunction-server/datajunction_server/api/graphql/scalars/__init__.py index 1ace61ee2..c3783aa4f 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/__init__.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/__init__.py @@ -28,7 +28,7 @@ class DateTimeJSONEncoder(json.JSONEncoder): JSON encoder that handles datetime objects """ - def default(self, obj): # pylint: disable=arguments-renamed + def default(self, obj): """ Check if there are datetime objects and serialize them as ISO format strings. @@ -46,7 +46,7 @@ class DateTimeJSONDecoder(json.JSONDecoder): def __init__(self, *args, **kwargs): json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) - def object_hook(self, source): # pylint: disable=method-hidden + def object_hook(self, source): """ Check if the string is in ISO 8601 format and convert to a datetime object if it is. @@ -80,7 +80,7 @@ def decode(cls, serialized: str) -> "Cursor": @strawberry.type -class PageInfo: # pylint: disable=too-few-public-methods +class PageInfo: """Metadata about a page in a connection.""" has_next_page: bool = field( @@ -98,14 +98,14 @@ class PageInfo: # pylint: disable=too-few-public-methods @strawberry.type -class Edge(Generic[GenericItemNode]): # pylint: disable=too-few-public-methods +class Edge(Generic[GenericItemNode]): """Metadata about an item in a connection.""" node: GenericItemNode @strawberry.type -class Connection(Generic[GenericItemNode]): # pylint: disable=too-few-public-methods +class Connection(Generic[GenericItemNode]): """ Pagination for a list of items. """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/availabilitystate.py b/datajunction-server/datajunction_server/api/graphql/scalars/availabilitystate.py index 363082e5c..bb1bcb1b0 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/availabilitystate.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/availabilitystate.py @@ -6,7 +6,7 @@ @strawberry.type -class PartitionAvailability: # pylint: disable=too-few-public-methods +class PartitionAvailability: """ Partition-level availability """ @@ -24,7 +24,7 @@ class PartitionAvailability: # pylint: disable=too-few-public-methods @strawberry.type -class AvailabilityState: # pylint: disable=too-few-public-methods +class AvailabilityState: """ A materialized table that is available for the node """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/catalog_engine.py b/datajunction-server/datajunction_server/api/graphql/scalars/catalog_engine.py index 26acb832f..59d519677 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/catalog_engine.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/catalog_engine.py @@ -10,14 +10,14 @@ @strawberry.experimental.pydantic.type(model=EngineInfo, all_fields=True) -class Engine: # pylint: disable=too-few-public-methods +class Engine: """ Database engine """ @strawberry.experimental.pydantic.type(model=CatalogInfo, all_fields=True) -class Catalog: # pylint: disable=too-few-public-methods +class Catalog: """ Class for a Catalog """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/column.py b/datajunction-server/datajunction_server/api/graphql/scalars/column.py index 0f939a06f..1a8658b74 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/column.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/column.py @@ -10,7 +10,7 @@ @strawberry.type -class AttributeTypeName: # pylint: disable=too-few-public-methods +class AttributeTypeName: """ Attribute type name. """ @@ -20,7 +20,7 @@ class AttributeTypeName: # pylint: disable=too-few-public-methods @strawberry.type -class Attribute: # pylint: disable=too-few-public-methods +class Attribute: """ Column attribute """ @@ -29,7 +29,7 @@ class Attribute: # pylint: disable=too-few-public-methods @strawberry.type -class NodeName: # pylint: disable=too-few-public-methods +class NodeName: """ Node name """ @@ -38,7 +38,7 @@ class NodeName: # pylint: disable=too-few-public-methods @strawberry.type -class Partition: # pylint: disable=too-few-public-methods +class Partition: """ A partition configuration for a column """ @@ -50,7 +50,7 @@ class Partition: # pylint: disable=too-few-public-methods @strawberry.type -class Column: # pylint: disable=too-few-public-methods +class Column: """ A column on a node """ @@ -61,4 +61,3 @@ class Column: # pylint: disable=too-few-public-methods attributes: Optional[List[Attribute]] dimension: Optional[NodeName] partition: Optional[Partition] - # order: Optional[int] diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/errors.py b/datajunction-server/datajunction_server/api/graphql/scalars/errors.py index 7ed6524f1..c467f9f06 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/errors.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/errors.py @@ -8,7 +8,7 @@ @strawberry.type -class DJError: # pylint: disable=too-few-public-methods +class DJError: """ A DJ error """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/materialization.py b/datajunction-server/datajunction_server/api/graphql/scalars/materialization.py index e234bd3df..ee226dc6f 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/materialization.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/materialization.py @@ -7,7 +7,7 @@ @strawberry.type -class PartitionBackfill: # pylint: disable=too-few-public-methods +class PartitionBackfill: """ Used for setting backfilled values """ @@ -23,7 +23,7 @@ class PartitionBackfill: # pylint: disable=too-few-public-methods @strawberry.type -class Backfill: # pylint: disable=too-few-public-methods +class Backfill: """ Materialization job backfill """ @@ -33,7 +33,7 @@ class Backfill: # pylint: disable=too-few-public-methods @strawberry.type -class MaterializationConfig: # pylint: disable=too-few-public-methods +class MaterializationConfig: """ Materialization config """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py b/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py index 3db7e5b02..78f5800e4 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py @@ -18,7 +18,7 @@ @strawberry.type -class Unit: # pylint: disable=too-few-public-methods +class Unit: """ Metric unit """ @@ -30,17 +30,15 @@ class Unit: # pylint: disable=too-few-public-methods @strawberry.experimental.pydantic.type(model=AggregationRule_, all_fields=True) -class AggregationRule: # pylint: disable=missing-class-docstring,too-few-public-methods - ... +class AggregationRule: ... @strawberry.experimental.pydantic.type(model=Measure_, all_fields=True) -class Measure: # pylint: disable=missing-class-docstring,too-few-public-methods - ... +class Measure: ... @strawberry.type -class ExtractedMeasures: # pylint: disable=too-few-public-methods +class ExtractedMeasures: """ extracted measures from metric """ @@ -51,7 +49,7 @@ class ExtractedMeasures: # pylint: disable=too-few-public-methods @strawberry.type -class MetricMetadata: # pylint: disable=too-few-public-methods +class MetricMetadata: """ Metric metadata output """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/node.py b/datajunction-server/datajunction_server/api/graphql/scalars/node.py index 6d6f116ed..fca4f4873 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/node.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/node.py @@ -41,7 +41,7 @@ @strawberry.type -class CubeElement: # pylint: disable=too-few-public-methods +class CubeElement: """ An element in a cube, either a metric or dimension """ @@ -53,7 +53,7 @@ class CubeElement: # pylint: disable=too-few-public-methods @strawberry.type -class DimensionLink: # pylint: disable=too-few-public-methods +class DimensionLink: """ A dimension link between a dimension and a node """ @@ -67,7 +67,7 @@ class DimensionLink: # pylint: disable=too-few-public-methods @strawberry.type -class DimensionAttribute: # pylint: disable=too-few-public-methods +class DimensionAttribute: """ A dimensional column attribute """ @@ -88,7 +88,6 @@ async def dimension_node(self, info: Info) -> "Node": if self._dimension_node: return self._dimension_node - # pylint: disable=import-outside-toplevel from datajunction_server.api.graphql.resolvers.nodes import get_node_by_name dimension_node_name = self.name.rsplit(".", 1)[0] @@ -121,7 +120,7 @@ def catalog(self, root: "DBNodeRevision") -> Optional[Catalog]: """ Catalog for the node """ - return Catalog.from_pydantic(root.catalog) # type: ignore # pylint: disable=no-member + return Catalog.from_pydantic(root.catalog) # type: ignore query: Optional[str] = None columns: List[Column] @@ -227,7 +226,7 @@ def cube_dimensions(self, root: "DBNodeRevision") -> List[DimensionAttribute]: @strawberry.type -class TagBase: # pylint: disable=too-few-public-methods +class TagBase: """ A DJ node tag without any referential fields """ @@ -240,7 +239,7 @@ class TagBase: # pylint: disable=too-few-public-methods @strawberry.type -class Node: # pylint: disable=too-few-public-methods +class Node: """ A DJ node """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/sql.py b/datajunction-server/datajunction_server/api/graphql/scalars/sql.py index 2810bcc75..22800e7b1 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/sql.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/sql.py @@ -53,7 +53,7 @@ async def column(self) -> str: @strawberry.type -class ColumnMetadata: # pylint: disable=too-few-public-methods +class ColumnMetadata: """ Column metadata for generated SQL """ @@ -65,7 +65,7 @@ class ColumnMetadata: # pylint: disable=too-few-public-methods @strawberry.type -class GeneratedSQL: # pylint: disable=too-few-public-methods +class GeneratedSQL: """ Generated SQL for a given node """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/tag.py b/datajunction-server/datajunction_server/api/graphql/scalars/tag.py index bee7deea0..728c09ed5 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/tag.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/tag.py @@ -9,7 +9,7 @@ @strawberry.type -class Tag(TagBase): # pylint: disable=too-few-public-methods +class Tag(TagBase): """ A DJ node tag with nodes """ diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/user.py b/datajunction-server/datajunction_server/api/graphql/scalars/user.py index f05fb112c..09585b42c 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/user.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/user.py @@ -22,7 +22,7 @@ class OAuthProvider(Enum): @strawberry.type -class User: # pylint: disable=too-few-public-methods +class User: """ A DataJunction User """ diff --git a/datajunction-server/datajunction_server/api/health.py b/datajunction-server/datajunction_server/api/health.py index b0b069df1..db4714feb 100644 --- a/datajunction-server/datajunction_server/api/health.py +++ b/datajunction-server/datajunction_server/api/health.py @@ -45,7 +45,7 @@ async def database_health(session: AsyncSession) -> HealthcheckStatus: HealthcheckStatus.OK if result == (1,) else HealthcheckStatus.FAILED ) return health_status - except Exception: # pylint: disable=broad-except # pragma: no cover + except Exception: # pragma: no cover return HealthcheckStatus.FAILED # pragma: no cover diff --git a/datajunction-server/datajunction_server/api/helpers.py b/datajunction-server/datajunction_server/api/helpers.py index 51e93d37c..fd26ce096 100644 --- a/datajunction-server/datajunction_server/api/helpers.py +++ b/datajunction-server/datajunction_server/api/helpers.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines """ Helpers for API endpoints """ @@ -66,7 +65,7 @@ COLUMN_NAME_REGEX = r"([A-Za-z0-9_\.]+)(\[[A-Za-z0-9_]+\])?" -async def get_node_namespace( # pylint: disable=too-many-arguments +async def get_node_namespace( session: AsyncSession, namespace: str, raise_if_not_exists: bool = True, @@ -85,7 +84,7 @@ async def get_node_namespace( # pylint: disable=too-many-arguments return node_namespace -async def get_node_by_name( # pylint: disable=too-many-arguments +async def get_node_by_name( session: AsyncSession, name: Optional[str], node_type: Optional[NodeType] = None, @@ -189,7 +188,7 @@ async def get_catalog_by_name(session: AsyncSession, name: str) -> Catalog: return catalog -async def get_query( # pylint: disable=too-many-arguments +async def get_query( session: AsyncSession, node_name: str, dimensions: List[str], @@ -203,9 +202,7 @@ async def get_query( # pylint: disable=too-many-arguments """ Get a query for a metric, dimensions, and filters """ - from datajunction_server.construction.build_v2 import ( # pylint: disable=import-outside-toplevel - QueryBuilder, - ) + from datajunction_server.construction.build_v2 import QueryBuilder node = await Node.get_by_name(session, node_name, raise_if_not_exists=True) build_criteria = get_default_criteria(node.current, engine) # type: ignore @@ -261,9 +258,7 @@ async def resolve_downstream_references( """ Find all node revisions with missing parent references to `node` and resolve them """ - from datajunction_server.internal.validation import ( # pylint: disable=import-outside-toplevel - validate_node_data, - ) + from datajunction_server.internal.validation import validate_node_data missing_parents = ( ( @@ -352,7 +347,7 @@ def map_dimensions_to_roles(dimensions: List[str]) -> Dict[str, str]: return {dim_rols[0]: dim_rols[1] for dim_rols in dimension_roles} -async def validate_cube( # pylint: disable=too-many-locals +async def validate_cube( session: AsyncSession, metric_names: List[str], dimension_names: List[str], @@ -576,7 +571,7 @@ async def find_existing_cube( ) for name in element_names: statement = statement.filter( - NodeRevision.cube_elements.any(Column.name == name), # type: ignore # pylint: disable=no-member + NodeRevision.cube_elements.any(Column.name == name), # type: ignore ).options( joinedload(Node.current).options( joinedload(NodeRevision.materializations), @@ -594,7 +589,7 @@ async def find_existing_cube( return None -async def build_sql_for_multiple_metrics( # pylint: disable=too-many-arguments,too-many-locals +async def build_sql_for_multiple_metrics( session: AsyncSession, metrics: List[str], dimensions: List[str], @@ -682,7 +677,7 @@ async def build_sql_for_multiple_metrics( # pylint: disable=too-many-arguments, access_control.add_request_by_node(cube) access_control.state = access.AccessControlState.INDIRECT access_control.raise_if_invalid_requests() - query_ast = build_materialized_cube_node( # pylint: disable=E1121 + query_ast = build_materialized_cube_node( metric_columns, dimension_columns, cube, @@ -758,7 +753,7 @@ async def build_sql_for_multiple_metrics( # pylint: disable=too-many-arguments, ) -async def query_event_stream( # pylint: disable=too-many-arguments +async def query_event_stream( query: QueryWithResults, request_headers: Optional[Dict[str, str]], query_service_client: QueryServiceClient, @@ -824,7 +819,7 @@ async def query_event_stream( # pylint: disable=too-many-arguments await asyncio.sleep(stream_delay) # pragma: no cover -async def build_sql_for_dj_query( # pylint: disable=too-many-locals # pragma: no cover +async def build_sql_for_dj_query( # pragma: no cover session: AsyncSession, query: str, access_control: access.AccessControl, diff --git a/datajunction-server/datajunction_server/api/main.py b/datajunction-server/datajunction_server/api/main.py index a8d2d952f..84b5f6461 100644 --- a/datajunction-server/datajunction_server/api/main.py +++ b/datajunction-server/datajunction_server/api/main.py @@ -5,7 +5,6 @@ # All the models need to be imported here so that SQLModel can define their # relationships at runtime without causing circular imports. # See https://sqlmodel.tiangolo.com/tutorial/code-structure/#make-circular-imports-work. -# pylint: disable=unused-import,ungrouped-imports import logging from http import HTTPStatus @@ -114,7 +113,7 @@ async def startup(): @app.exception_handler(DJException) -async def dj_exception_handler( # pylint: disable=unused-argument +async def dj_exception_handler( request: Request, exc: DJException, ) -> JSONResponse: diff --git a/datajunction-server/datajunction_server/api/materializations.py b/datajunction-server/datajunction_server/api/materializations.py index 1fc2efef4..82ade21a6 100644 --- a/datajunction-server/datajunction_server/api/materializations.py +++ b/datajunction-server/datajunction_server/api/materializations.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines,too-many-arguments """ Node materialization related APIs. """ @@ -80,7 +79,7 @@ def materialization_jobs_info() -> JSONResponse: status_code=201, name="Insert or Update a Materialization for a Node", ) -async def upsert_materialization( # pylint: disable=too-many-locals +async def upsert_materialization( node_name: str, data: UpsertMaterialization | UpsertCubeMaterialization, *, @@ -88,7 +87,7 @@ async def upsert_materialization( # pylint: disable=too-many-locals request: Request, query_service_client: QueryServiceClient = Depends(get_query_service_client), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> JSONResponse: @@ -375,7 +374,7 @@ async def deactivate_node_materializations( status_code=201, name="Kick off a backfill run for a configured materialization", ) -async def run_materialization_backfill( # pylint: disable=too-many-locals +async def run_materialization_backfill( node_name: str, materialization_name: str, backfill_partitions: List[PartitionBackfill], diff --git a/datajunction-server/datajunction_server/api/measures.py b/datajunction-server/datajunction_server/api/measures.py index 4fabeefa1..87c519e40 100644 --- a/datajunction-server/datajunction_server/api/measures.py +++ b/datajunction-server/datajunction_server/api/measures.py @@ -88,7 +88,7 @@ async def list_measures( statement = select(Measure.name) if prefix: statement = statement.where( - Measure.name.like(f"{prefix}%"), # type: ignore # pylint: disable=no-member + Measure.name.like(f"{prefix}%"), # type: ignore ) return (await session.execute(statement)).scalars().all() diff --git a/datajunction-server/datajunction_server/api/metrics.py b/datajunction-server/datajunction_server/api/metrics.py index e5a6597c8..329bb61ed 100644 --- a/datajunction-server/datajunction_server/api/metrics.py +++ b/datajunction-server/datajunction_server/api/metrics.py @@ -66,7 +66,7 @@ async def list_metrics( *, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> List[str]: @@ -124,7 +124,7 @@ async def get_common_dimensions( input_errors = [] statement = ( select(Node) - .where(Node.name.in_(metric)) # type: ignore # pylint: disable=no-member + .where(Node.name.in_(metric)) # type: ignore .where(is_(Node.deactivated_at, None)) ) metric_nodes = (await session.execute(statement)).scalars().all() diff --git a/datajunction-server/datajunction_server/api/namespaces.py b/datajunction-server/datajunction_server/api/namespaces.py index 68617a799..2946f2a4e 100644 --- a/datajunction-server/datajunction_server/api/namespaces.py +++ b/datajunction-server/datajunction_server/api/namespaces.py @@ -104,7 +104,7 @@ async def create_node_namespace( async def list_namespaces( session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> List[NamespaceOutput]: diff --git a/datajunction-server/datajunction_server/api/nodes.py b/datajunction-server/datajunction_server/api/nodes.py index 2fb34aa31..4585cb2d4 100644 --- a/datajunction-server/datajunction_server/api/nodes.py +++ b/datajunction-server/datajunction_server/api/nodes.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines,too-many-arguments """ Node related APIs. """ @@ -236,7 +235,7 @@ async def list_nodes( *, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> List[str]: @@ -267,7 +266,7 @@ async def list_all_nodes_with_details( *, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> List[NodeIndexItem]: @@ -420,7 +419,7 @@ async def create_source( current_user: User = Depends(get_and_update_current_user), request: Request, query_service_client: QueryServiceClient = Depends(get_query_service_client), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), background_tasks: BackgroundTasks, @@ -533,7 +532,7 @@ async def create_node( current_user: User = Depends(get_and_update_current_user), query_service_client: QueryServiceClient = Depends(get_query_service_client), background_tasks: BackgroundTasks, - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> NodeOutput: @@ -639,7 +638,7 @@ async def create_cube( query_service_client: QueryServiceClient = Depends(get_query_service_client), current_user: User = Depends(get_and_update_current_user), background_tasks: BackgroundTasks, - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> NodeOutput: @@ -756,7 +755,7 @@ async def register_table( response_model=NodeOutput, status_code=201, ) -async def register_view( # pylint: disable=too-many-locals +async def register_view( catalog: str, schema_: str, view: str, @@ -842,7 +841,7 @@ async def link_dimension( name: str, column: str, dimension: str, - dimension_column: Optional[str] = None, # pylint: disable=unused-argument + dimension_column: Optional[str] = None, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), ) -> JSONResponse: @@ -1042,7 +1041,7 @@ async def remove_reference_dimension_link( @router.post("/nodes/{node_name}/link/", status_code=201) -async def add_complex_dimension_link( # pylint: disable=too-many-locals +async def add_complex_dimension_link( node_name: str, link_input: JoinLinkInput, session: AsyncSession = Depends(get_session), @@ -1075,7 +1074,7 @@ async def add_complex_dimension_link( # pylint: disable=too-many-locals @router.delete("/nodes/{node_name}/link/", status_code=201) -async def remove_complex_dimension_link( # pylint: disable=too-many-locals +async def remove_complex_dimension_link( node_name: str, link_identifier: LinkDimensionIdentifier, session: AsyncSession = Depends(get_session), @@ -1095,9 +1094,9 @@ async def remove_complex_dimension_link( # pylint: disable=too-many-locals @router.delete("/nodes/{name}/columns/{column}/", status_code=201) async def delete_dimension_link( name: str, - column: str, # pylint: disable=unused-argument + column: str, dimension: str, - dimension_column: Optional[str] = None, # pylint: disable=unused-argument + dimension_column: Optional[str] = None, session: AsyncSession = Depends(get_session), current_user: User = Depends(get_and_update_current_user), ) -> JSONResponse: @@ -1168,7 +1167,7 @@ async def tags_node( response_model=NodeOutput, status_code=201, ) -async def refresh_source_node( # pylint: disable=too-many-locals +async def refresh_source_node( name: str, *, session: AsyncSession = Depends(get_session), @@ -1334,7 +1333,7 @@ async def update_node( query_service_client: QueryServiceClient = Depends(get_query_service_client), current_user: User = Depends(get_and_update_current_user), background_tasks: BackgroundTasks, - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ) -> NodeOutput: @@ -1579,7 +1578,7 @@ async def set_column_display_name( status_code=201, name="Set Node Column as Partition", ) -async def set_column_partition( # pylint: disable=too-many-locals +async def set_column_partition( node_name: str, column_name: str, input_partition: PartitionInput, diff --git a/datajunction-server/datajunction_server/api/sql.py b/datajunction-server/datajunction_server/api/sql.py index a87214a9d..c4151f4cc 100644 --- a/datajunction-server/datajunction_server/api/sql.py +++ b/datajunction-server/datajunction_server/api/sql.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-arguments """ SQL related APIs. """ @@ -67,12 +66,12 @@ async def get_measures_sql_for_cube_v2( "for the metrics and dimensions in the cube" ), ), - settings: Settings = Depends(get_settings), # pylint: disable=redefined-outer-name + settings: Settings = Depends(get_settings), session: AsyncSession = Depends(get_session), engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: Optional[User] = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), use_materialized: bool = True, @@ -88,7 +87,7 @@ async def get_measures_sql_for_cube_v2( and others are aggregations on measures in parent node B, this endpoint will generate two measures queries, one for A and one for B. """ - from datajunction_server.construction.build_v2 import ( # pylint: disable=import-outside-toplevel,line-too-long + from datajunction_server.construction.build_v2 import ( get_measures_query, ) @@ -111,7 +110,7 @@ async def get_measures_sql_for_cube_v2( return measures_query -async def build_and_save_node_sql( # pylint: disable=too-many-locals +async def build_and_save_node_sql( node_name: str, dimensions: List[str] = Query([]), filters: List[str] = Query([]), @@ -228,7 +227,7 @@ async def build_and_save_node_sql( # pylint: disable=too-many-locals return query_request -async def get_node_sql( # pylint: disable=too-many-locals +async def get_node_sql( node_name: str, dimensions: List[str] = Query([]), filters: List[str] = Query([]), @@ -239,7 +238,7 @@ async def get_node_sql( # pylint: disable=too-many-locals engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User, - validate_access: access.ValidateAccessFn, # pylint: disable=redefined-outer-name + validate_access: access.ValidateAccessFn, background_tasks: BackgroundTasks, ignore_errors: bool = True, use_materialized: bool = True, @@ -321,7 +320,7 @@ async def get_node_sql( # pylint: disable=too-many-locals response_model=TranslatedSQL, name="Get SQL For A Node", ) -async def get_sql( # pylint: disable=too-many-locals +async def get_sql( node_name: str, dimensions: List[str] = Query([]), filters: List[str] = Query([]), @@ -332,7 +331,7 @@ async def get_sql( # pylint: disable=too-many-locals engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), background_tasks: BackgroundTasks, @@ -361,7 +360,7 @@ async def get_sql( # pylint: disable=too-many-locals @router.get("/sql/", response_model=TranslatedSQL, name="Get SQL For Metrics") -async def get_sql_for_metrics( # pylint: disable=too-many-locals +async def get_sql_for_metrics( metrics: List[str] = Query([]), dimensions: List[str] = Query([]), filters: List[str] = Query([]), @@ -372,7 +371,7 @@ async def get_sql_for_metrics( # pylint: disable=too-many-locals engine_name: Optional[str] = None, engine_version: Optional[str] = None, current_user: User = Depends(get_and_update_current_user), - validate_access: access.ValidateAccessFn = Depends( # pylint: disable=W0621 + validate_access: access.ValidateAccessFn = Depends( validate_access, ), ignore_errors: Optional[bool] = True, @@ -455,7 +454,7 @@ async def get_sql_for_metrics( # pylint: disable=too-many-locals ) -async def build_and_save_sql_for_metrics( # pylint: disable=too-many-arguments,too-many-locals +async def build_and_save_sql_for_metrics( session: AsyncSession, metrics: List[str], dimensions: List[str], diff --git a/datajunction-server/datajunction_server/api/tags.py b/datajunction-server/datajunction_server/api/tags.py index a1869ad3d..b4ab05abb 100644 --- a/datajunction-server/datajunction_server/api/tags.py +++ b/datajunction-server/datajunction_server/api/tags.py @@ -35,7 +35,7 @@ async def get_tags_by_name( """ Retrieves a list of tags by name """ - statement = select(Tag).where(Tag.name.in_(names)) # type: ignore # pylint: disable=no-member + statement = select(Tag).where(Tag.name.in_(names)) # type: ignore tags = (await session.execute(statement)).scalars().all() difference = set(names) - {tag.name for tag in tags} if difference: diff --git a/datajunction-server/datajunction_server/api/users.py b/datajunction-server/datajunction_server/api/users.py index fcb5332a4..26a628c94 100644 --- a/datajunction-server/datajunction_server/api/users.py +++ b/datajunction-server/datajunction_server/api/users.py @@ -72,15 +72,15 @@ async def list_users_with_activity( statement = ( select( User.username, - func.count(History.id).label("count"), # pylint: disable=not-callable + func.count(History.id).label("count"), ) .join( - History, # pylint: disable=superfluous-parens + History, onclause=(User.username == History.user), isouter=True, ) .group_by(User.username) - .order_by(func.count(History.id).desc()) # pylint: disable=not-callable + .order_by(func.count(History.id).desc()) ) result = await session.execute(statement) return [ diff --git a/datajunction-server/datajunction_server/config.py b/datajunction-server/datajunction_server/config.py index 00dd780b0..d92511e92 100644 --- a/datajunction-server/datajunction_server/config.py +++ b/datajunction-server/datajunction_server/config.py @@ -17,9 +17,7 @@ pass -class Settings( - BaseSettings, -): # pylint: disable=too-few-public-methods #pragma: no cover +class Settings(BaseSettings): # pragma: no cover """ DataJunction configuration. """ diff --git a/datajunction-server/datajunction_server/construction/build.py b/datajunction-server/datajunction_server/construction/build.py index 2a749a262..dcfcff547 100755 --- a/datajunction-server/datajunction_server/construction/build.py +++ b/datajunction-server/datajunction_server/construction/build.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-arguments,too-many-locals,too-many-nested-blocks,too-many-branches,R0401,too-many-lines,protected-access,line-too-long """Functions for building DJ node queries""" import collections @@ -91,7 +90,7 @@ def rename_columns( ): column_ref = expression.alias_or_name.identifier() if column_ref in node_columns: # type: ignore - alias_name = f"{node.name}{SEPARATOR}{column_ref}" # type: ignore # pragma: no cover # pylint: disable=line-too-long + alias_name = f"{node.name}{SEPARATOR}{column_ref}" # type: ignore # pragma: no cover expression.set_semantic_entity(alias_name) # pragma: no cover else: expression.set_semantic_entity(from_amenable_name(column_ref)) @@ -104,15 +103,13 @@ def rename_columns( col.alias = None if built_ast.select.group_by: - for i in range( # pylint: disable=consider-using-enumerate # pragma: no cover + for i in range( # pragma: no cover len(built_ast.select.group_by), ): if hasattr(built_ast.select.group_by[i], "alias"): # pragma: no cover built_ast.select.group_by[i] = ast.Column( name=built_ast.select.group_by[i].name, # type: ignore - # pylint:disable=protected-access _type=built_ast.select.group_by[i].type, # type: ignore - # pylint:disable=protected-access _table=built_ast.select.group_by[i]._table, # type: ignore ) built_ast.select.group_by[i].alias = None @@ -155,13 +152,13 @@ async def validate_shared_dimensions( ) -async def build_metric_nodes( # pylint: disable=too-many-statements +async def build_metric_nodes( session: AsyncSession, metric_nodes: List[Node], filters: List[str], dimensions: List[str], - orderby: List[str], # pylint: disable=unused-argument - limit: Optional[int] = None, # pylint: disable=unused-argument + orderby: List[str], + limit: Optional[int] = None, engine_name: Optional[str] = None, engine_version: Optional[str] = None, build_criteria: Optional[BuildCriteria] = None, @@ -180,9 +177,7 @@ async def build_metric_nodes( # pylint: disable=too-many-statements (e) Join together the transforms on the shared dimensions (f) Select all the requested metrics and dimensions in the final SELECT """ - from datajunction_server.construction.build_v2 import ( # pylint: disable=import-outside-toplevel - CubeQueryBuilder, - ) + from datajunction_server.construction.build_v2 import CubeQueryBuilder engine = ( await get_engine(session, engine_name, engine_version) @@ -244,7 +239,7 @@ def build_materialized_cube_node( cube_config = GenericCubeConfig.parse_obj(materialization_config.config) if materialization_config.name == "default": - # TODO: remove after we migrate old Druid materializations # pylint: disable=fixme + # TODO: remove after we migrate old Druid materializations selected_metric_keys = [ col.name for col in selected_metrics ] # pragma: no cover @@ -316,7 +311,6 @@ def build_materialized_cube_node( filter_asts.append(temp_select.where) if filter_asts: # pragma: no cover - # pylint: disable=no-value-for-parameter combined_ast.select.where = ast.BinaryOp.And(*filter_asts) # Add orderby diff --git a/datajunction-server/datajunction_server/construction/build_v2.py b/datajunction-server/datajunction_server/construction/build_v2.py index 26c84c0ff..7338796a4 100644 --- a/datajunction-server/datajunction_server/construction/build_v2.py +++ b/datajunction-server/datajunction_server/construction/build_v2.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-arguments,too-many-locals,too-many-nested-blocks,too-many-branches,R0401,too-many-lines,protected-access,line-too-long """Building node SQL functions""" import collections @@ -91,7 +90,7 @@ class DimensionJoin: node_query: Optional[ast.Query] = None -async def get_measures_query( # pylint: disable=too-many-locals +async def get_measures_query( session: AsyncSession, metrics: List[str], dimensions: List[str], @@ -114,12 +113,12 @@ async def get_measures_query( # pylint: disable=too-many-locals and others are aggregations on measures in parent node B, this function will return a dictionary that maps A to the measures query for A, and B to the measures query for B. """ - from datajunction_server.api.helpers import ( # pylint: disable=import-outside-toplevel + from datajunction_server.api.helpers import ( assemble_column_metadata, check_dimension_attributes_exist, check_metrics_exist, ) - from datajunction_server.construction.build import ( # pylint: disable=import-outside-toplevel + from datajunction_server.construction.build import ( group_metrics_by_parent, metrics_to_measures, rename_columns, @@ -338,7 +337,7 @@ def build_preaggregate_query( return final_query -class QueryBuilder: # pylint: disable=too-many-instance-attributes,too-many-public-methods +class QueryBuilder: """ This class allows users to configure building node SQL by incrementally building out the build configuration, including adding filters, dimensions, ordering, and limit @@ -697,7 +696,7 @@ async def build_dimension_node_joins(self, node_ast, node_alias): link = cast(DimensionLink, link) if all( dim in link.foreign_keys_reversed for dim in requested_dimensions - ): # pylint: disable=line-too-long # pragma: no cover + ): # pragma: no cover continue # pragma: no cover if link.dimension.name in self.cte_mapping: @@ -838,7 +837,7 @@ async def find_dimension_node_joins( return dimension_node_joins -class CubeQueryBuilder: # pylint: disable=too-many-instance-attributes +class CubeQueryBuilder: """ This class allows users to configure building cube SQL (retrieving SQL for multiple metrics + dimensions) through settings like adding filters, dimensions, ordering, and limit @@ -1080,7 +1079,7 @@ async def build_measures_queries(self): """ Build the metrics' queries grouped by parent """ - from datajunction_server.construction.build import ( # pylint: disable=import-outside-toplevel,line-too-long + from datajunction_server.construction.build import ( group_metrics_by_parent, ) @@ -1541,7 +1540,7 @@ def build_join_for_link( return join_ast -async def build_ast( # pylint: disable=too-many-arguments,too-many-locals,too-many-statements +async def build_ast( session: AsyncSession, node: NodeRevision, query: ast.Query, @@ -1576,7 +1575,7 @@ async def build_ast( # pylint: disable=too-many-arguments,too-many-locals,too-m else: await query.compile(context) - query.bake_ctes() # pylint: disable=W0212 + query.bake_ctes() await refresh_if_needed(session, node, ["dimension_links"]) new_cte_mapping: Dict[str, ast.Query] = {} @@ -1627,7 +1626,7 @@ async def build_ast( # pylint: disable=too-many-arguments,too-many-locals,too-m ) query_ast = ast.Table( # type: ignore reference_cte.alias, # type: ignore - _columns=reference_cte._columns, # pylint: disable=protected-access + _columns=reference_cte._columns, _dj_node=referenced_node, ) else: @@ -1671,7 +1670,7 @@ async def build_ast( # pylint: disable=too-many-arguments,too-many-locals,too-m and col.table.dj_node and col.table.dj_node.name == referenced_node.name ): - col._table = query_ast # pylint: disable=protected-access + col._table = query_ast # Apply pushdown filters if possible apply_filters_to_node(node, query, to_filter_asts(filters)) diff --git a/datajunction-server/datajunction_server/construction/dimensions.py b/datajunction-server/datajunction_server/construction/dimensions.py index b3c545116..e92f27fda 100644 --- a/datajunction-server/datajunction_server/construction/dimensions.py +++ b/datajunction-server/datajunction_server/construction/dimensions.py @@ -22,7 +22,7 @@ from datajunction_server.utils import SEPARATOR -async def build_dimensions_from_cube_query( # pylint: disable=too-many-arguments,too-many-locals +async def build_dimensions_from_cube_query( session: AsyncSession, cube: NodeRevision, dimensions: List[str], diff --git a/datajunction-server/datajunction_server/construction/dj_query.py b/datajunction-server/datajunction_server/construction/dj_query.py index 484485849..01d64084a 100644 --- a/datajunction-server/datajunction_server/construction/dj_query.py +++ b/datajunction-server/datajunction_server/construction/dj_query.py @@ -30,7 +30,7 @@ def selects_only_metrics(select: ast.Select) -> bool: ) -async def resolve_metric_queries( # pylint: disable=R0914,R0912,R0915 +async def resolve_metric_queries( session: AsyncSession, tree: ast.Query, ctx: ast.CompileContext, @@ -197,7 +197,7 @@ def find_all_other( node.apply(lambda n: find_all_other(n, touched_nodes, node_map)) -async def resolve_all( # pylint: disable=R0914,W0640 +async def resolve_all( session: AsyncSession, ctx: ast.CompileContext, tree: ast.Query, diff --git a/datajunction-server/datajunction_server/construction/utils.py b/datajunction-server/datajunction_server/construction/utils.py index 167196708..63c873329 100755 --- a/datajunction-server/datajunction_server/construction/utils.py +++ b/datajunction-server/datajunction_server/construction/utils.py @@ -26,7 +26,7 @@ async def get_dj_node( """Return the DJ Node with a given name from a set of node types""" query = select(Node).filter(Node.name == node_name) if kinds: - query = query.filter(Node.type.in_(kinds)) # type: ignore # pylint: disable=no-member + query = query.filter(Node.type.in_(kinds)) # type: ignore match = None try: match = ( @@ -59,7 +59,7 @@ async def try_get_dj_node( kinds: Optional[Set[NodeType]] = None, ) -> Optional[Node]: "wraps get dj node to return None if no node is found" - from datajunction_server.sql.parsing.ast import Column # pylint: disable=C0415 + from datajunction_server.sql.parsing.ast import Column if isinstance(name, Column): if name.name.namespace is not None: # pragma: no cover @@ -76,9 +76,7 @@ def to_namespaced_name(name: str) -> "Name": """ Builds a namespaced name from a string """ - from datajunction_server.sql.parsing.ast import ( # pylint: disable=import-outside-toplevel - Name, - ) + from datajunction_server.sql.parsing.ast import Name chunked = name.split(".") chunked.reverse() diff --git a/datajunction-server/datajunction_server/database/attributetype.py b/datajunction-server/datajunction_server/database/attributetype.py index 1b0619410..008b41d62 100644 --- a/datajunction-server/datajunction_server/database/attributetype.py +++ b/datajunction-server/datajunction_server/database/attributetype.py @@ -14,9 +14,7 @@ from datajunction_server.database.column import Column -class AttributeType( - Base, -): # pylint: disable=too-few-public-methods,unsubscriptable-object +class AttributeType(Base): """ Available attribute types for column metadata. """ @@ -83,9 +81,7 @@ async def create( return attribute_type -class ColumnAttribute( - Base, -): # pylint: disable=too-few-public-methods,unsubscriptable-object +class ColumnAttribute(Base): """ Column attributes. """ diff --git a/datajunction-server/datajunction_server/database/availabilitystate.py b/datajunction-server/datajunction_server/database/availabilitystate.py index 26ff8c423..5da82659c 100644 --- a/datajunction-server/datajunction_server/database/availabilitystate.py +++ b/datajunction-server/datajunction_server/database/availabilitystate.py @@ -1,5 +1,4 @@ """Availability state database schema.""" -# pylint: disable=unsubscriptable-object from datetime import datetime, timezone from functools import partial @@ -14,7 +13,7 @@ from datajunction_server.typing import UTCDatetime -class AvailabilityState(Base): # pylint: disable=too-few-public-methods +class AvailabilityState(Base): """ The availability of materialized data for a node """ @@ -68,17 +67,17 @@ class AvailabilityState(Base): # pylint: disable=too-few-public-methods def is_available( self, - criteria: Optional[BuildCriteria] = None, # pylint: disable=unused-argument + criteria: Optional[BuildCriteria] = None, ) -> bool: # pragma: no cover """ Determine whether an availability state is useable given criteria """ - # TODO: we should evaluate this availability state against the criteria. # pylint: disable=fixme + # TODO: we should evaluate this availability state against the criteria. # Remember that VTTS can be also evaluated at runtime dependency. return True -class NodeAvailabilityState(Base): # pylint: disable=too-few-public-methods +class NodeAvailabilityState(Base): """ Join table for availability state """ diff --git a/datajunction-server/datajunction_server/database/backfill.py b/datajunction-server/datajunction_server/database/backfill.py index 9b6e647ef..ed8a3d18f 100644 --- a/datajunction-server/datajunction_server/database/backfill.py +++ b/datajunction-server/datajunction_server/database/backfill.py @@ -1,4 +1,3 @@ -# pylint: disable=unsubscriptable-object """Backfill database schema.""" from typing import TYPE_CHECKING, List, Optional @@ -13,7 +12,7 @@ from datajunction_server.database.materialization import Materialization -class Backfill(Base): # type: ignore # pylint: disable=too-few-public-methods +class Backfill(Base): # type: ignore """ A backfill run is linked to a materialization config, where users provide the range (of a temporal partition) to backfill for the node. diff --git a/datajunction-server/datajunction_server/database/catalog.py b/datajunction-server/datajunction_server/database/catalog.py index 81b30801a..381085a93 100644 --- a/datajunction-server/datajunction_server/database/catalog.py +++ b/datajunction-server/datajunction_server/database/catalog.py @@ -56,7 +56,7 @@ def __hash__(self) -> int: return hash(self.id) -class CatalogEngines(Base): # type: ignore # pylint: disable=too-few-public-methods +class CatalogEngines(Base): # type: ignore """ Join table for catalogs and engines. """ diff --git a/datajunction-server/datajunction_server/database/collection.py b/datajunction-server/datajunction_server/database/collection.py index 6e23d5091..577219440 100644 --- a/datajunction-server/datajunction_server/database/collection.py +++ b/datajunction-server/datajunction_server/database/collection.py @@ -15,7 +15,7 @@ from datajunction_server.typing import UTCDatetime -class Collection(Base): # pylint: disable=too-few-public-methods +class Collection(Base): """ A collection of nodes """ @@ -71,7 +71,7 @@ async def get_by_name( return collection -class CollectionNodes(Base): # type: ignore # pylint: disable=too-few-public-methods +class CollectionNodes(Base): # type: ignore """ Join table for collections and nodes. """ diff --git a/datajunction-server/datajunction_server/database/column.py b/datajunction-server/datajunction_server/database/column.py index 12f7635db..8ea5b9a38 100644 --- a/datajunction-server/datajunction_server/database/column.py +++ b/datajunction-server/datajunction_server/database/column.py @@ -114,8 +114,7 @@ def has_attribute(self, attribute_name: str) -> bool: Whether the given attribute is set on this column. """ return any( - attr.attribute_type.name == attribute_name - for attr in self.attributes # pylint: disable=not-an-iterable + attr.attribute_type.name == attribute_name for attr in self.attributes ) def attribute_names(self) -> list[str]: @@ -129,8 +128,7 @@ def has_attributes_besides(self, attribute_name: str) -> bool: Whether the column has any attribute besides the one specified. """ return any( - attr.attribute_type.name != attribute_name - for attr in self.attributes # pylint: disable=not-an-iterable + attr.attribute_type.name != attribute_name for attr in self.attributes ) def __hash__(self) -> int: diff --git a/datajunction-server/datajunction_server/database/database.py b/datajunction-server/datajunction_server/database/database.py index 36f5fc80c..4f2008d6f 100644 --- a/datajunction-server/datajunction_server/database/database.py +++ b/datajunction-server/datajunction_server/database/database.py @@ -16,7 +16,7 @@ from datajunction_server.database.column import Column -class Database(Base): # pylint: disable=too-few-public-methods +class Database(Base): """ A database. @@ -103,7 +103,7 @@ def identifier( """ # Catalogs will soon be required and this return can be simplified return ( - self.catalog.name if self.catalog else None, # pylint: disable=no-member + self.catalog.name if self.catalog else None, self.schema_, self.table, ) @@ -112,7 +112,7 @@ def __hash__(self): return hash(self.id) -class TableColumns(Base): # pylint: disable=too-few-public-methods +class TableColumns(Base): """ Join table for table columns. """ diff --git a/datajunction-server/datajunction_server/database/dimensionlink.py b/datajunction-server/datajunction_server/database/dimensionlink.py index d687162f8..f37ceb091 100644 --- a/datajunction-server/datajunction_server/database/dimensionlink.py +++ b/datajunction-server/datajunction_server/database/dimensionlink.py @@ -16,7 +16,7 @@ from datajunction_server.sql.parsing.backends.antlr4 import ast -class DimensionLink(Base): # pylint: disable=too-few-public-methods +class DimensionLink(Base): """ The join definition between a given node (source, dimension, or transform) and a dimension node. @@ -90,7 +90,6 @@ def join_sql_ast(self) -> "ast.Query": """ The join query AST for this dimension link """ - # pylint: disable=import-outside-toplevel from datajunction_server.sql.parsing.backends.antlr4 import parse return parse( @@ -113,7 +112,6 @@ def foreign_key_mapping(self) -> Dict["ast.Column", "ast.Column"]: returns a mapping between the foreign keys on the node and the primary keys of the dimension based on the join SQL. """ - # pylint: disable=import-outside-toplevel from datajunction_server.sql.parsing.backends.antlr4 import ast # Find equality comparions (i.e., fact.order_id = dim.order_id) diff --git a/datajunction-server/datajunction_server/database/engine.py b/datajunction-server/datajunction_server/database/engine.py index 73dbba762..5cc11bd21 100644 --- a/datajunction-server/datajunction_server/database/engine.py +++ b/datajunction-server/datajunction_server/database/engine.py @@ -10,7 +10,7 @@ from datajunction_server.models.engine import Dialect -class Engine(Base): # pylint: disable=too-few-public-methods +class Engine(Base): """ A query engine. """ diff --git a/datajunction-server/datajunction_server/database/history.py b/datajunction-server/datajunction_server/database/history.py index d41a9f77c..6a5901e2d 100644 --- a/datajunction-server/datajunction_server/database/history.py +++ b/datajunction-server/datajunction_server/database/history.py @@ -48,7 +48,7 @@ class EntityType(StrEnum): TAG = "tag" -class History(Base): # pylint: disable=too-few-public-methods +class History(Base): """ An event to store as part of the server's activity history """ diff --git a/datajunction-server/datajunction_server/database/materialization.py b/datajunction-server/datajunction_server/database/materialization.py index 4616bee9b..930792843 100644 --- a/datajunction-server/datajunction_server/database/materialization.py +++ b/datajunction-server/datajunction_server/database/materialization.py @@ -30,7 +30,7 @@ from datajunction_server.database.node import NodeRevision -class Materialization(Base): # pylint: disable=too-few-public-methods +class Materialization(Base): """ Materialization configured for a node. """ @@ -107,9 +107,7 @@ async def get_by_names( """ Get materializations by name and node revision id. """ - from datajunction_server.database.node import ( # pylint: disable=import-outside-toplevel - NodeRevision, - ) + from datajunction_server.database.node import NodeRevision statement = ( select(cls) diff --git a/datajunction-server/datajunction_server/database/measure.py b/datajunction-server/datajunction_server/database/measure.py index a2452416a..7c99f48be 100644 --- a/datajunction-server/datajunction_server/database/measure.py +++ b/datajunction-server/datajunction_server/database/measure.py @@ -11,7 +11,7 @@ from datajunction_server.models.measure import AggregationRule -class Measure(Base): # type: ignore # pylint: disable=too-few-public-methods +class Measure(Base): # type: ignore """ Measure class. diff --git a/datajunction-server/datajunction_server/database/metricmetadata.py b/datajunction-server/datajunction_server/database/metricmetadata.py index 60c7c3629..4d70fda2e 100644 --- a/datajunction-server/datajunction_server/database/metricmetadata.py +++ b/datajunction-server/datajunction_server/database/metricmetadata.py @@ -14,9 +14,7 @@ ) -class MetricMetadata( - Base, -): # pylint: disable=too-few-public-methods,unsubscriptable-object +class MetricMetadata(Base): """ Additional metric metadata """ diff --git a/datajunction-server/datajunction_server/database/namespace.py b/datajunction-server/datajunction_server/database/namespace.py index 33e8a3688..9282dc7cf 100644 --- a/datajunction-server/datajunction_server/database/namespace.py +++ b/datajunction-server/datajunction_server/database/namespace.py @@ -16,7 +16,7 @@ from datajunction_server.typing import UTCDatetime -class NodeNamespace(Base): # pylint: disable=too-few-public-methods +class NodeNamespace(Base): """ A node namespace """ @@ -42,7 +42,7 @@ async def get_all_with_node_count(cls, session: AsyncSession): statement = ( select( NodeNamespace.namespace, - func.count(Node.id).label("num_nodes"), # pylint: disable=not-callable + func.count(Node.id).label("num_nodes"), ) .join( Node, @@ -96,7 +96,7 @@ async def list_nodes( select(Node) .where( or_( - Node.namespace.like(f"{namespace}.%"), # pylint: disable=no-member + Node.namespace.like(f"{namespace}.%"), Node.namespace == namespace, ), Node.type == node_type if node_type else True, @@ -160,7 +160,7 @@ async def list_all_nodes( select(Node) .where( or_( - Node.namespace.like(f"{namespace}.%"), # pylint: disable=no-member + Node.namespace.like(f"{namespace}.%"), Node.namespace == namespace, ), ) diff --git a/datajunction-server/datajunction_server/database/node.py b/datajunction-server/datajunction_server/database/node.py index 0610f599a..20babae0e 100644 --- a/datajunction-server/datajunction_server/database/node.py +++ b/datajunction-server/datajunction_server/database/node.py @@ -62,7 +62,7 @@ from datajunction_server.database.dimensionlink import DimensionLink -class NodeRelationship(Base): # pylint: disable=too-few-public-methods +class NodeRelationship(Base): """ Join table for self-referential many-to-many relationships between nodes. """ @@ -84,7 +84,7 @@ class NodeRelationship(Base): # pylint: disable=too-few-public-methods ) -class CubeRelationship(Base): # pylint: disable=too-few-public-methods +class CubeRelationship(Base): """ Join table for many-to-many relationships between cube nodes and metric/dimension nodes. """ @@ -102,7 +102,7 @@ class CubeRelationship(Base): # pylint: disable=too-few-public-methods ) -class BoundDimensionsRelationship(Base): # pylint: disable=too-few-public-methods +class BoundDimensionsRelationship(Base): """ Join table for many-to-many relationships between metric nodes and parent nodes for dimensions that are required. @@ -127,7 +127,7 @@ class BoundDimensionsRelationship(Base): # pylint: disable=too-few-public-metho ) -class MissingParent(Base): # pylint: disable=too-few-public-methods +class MissingParent(Base): """ A missing parent node """ @@ -145,7 +145,7 @@ class MissingParent(Base): # pylint: disable=too-few-public-methods ) -class NodeMissingParents(Base): # pylint: disable=too-few-public-methods +class NodeMissingParents(Base): """ Join table for missing parents """ @@ -168,7 +168,7 @@ class NodeMissingParents(Base): # pylint: disable=too-few-public-methods ) -class Node(Base): # pylint: disable=too-few-public-methods +class Node(Base): """ Node that acts as an umbrella for all node revisions """ @@ -384,7 +384,7 @@ async def get_by_id( return node # pragma: no cover @classmethod - async def find( # pylint: disable=keyword-arg-before-vararg + async def find( cls, session: AsyncSession, prefix: Optional[str] = None, @@ -397,7 +397,7 @@ async def find( # pylint: disable=keyword-arg-before-vararg statement = select(Node).where(is_(Node.deactivated_at, None)) if prefix: statement = statement.where( - Node.name.like(f"{prefix}%"), # type: ignore # pylint: disable=no-member + Node.name.like(f"{prefix}%"), # type: ignore ) if node_type: statement = statement.where(Node.type == node_type) @@ -405,7 +405,7 @@ async def find( # pylint: disable=keyword-arg-before-vararg return result.unique().scalars().all() @classmethod - async def find_by( # pylint: disable=keyword-arg-before-vararg,too-many-locals + async def find_by( cls, session: AsyncSession, names: Optional[List[str]] = None, @@ -417,7 +417,7 @@ async def find_by( # pylint: disable=keyword-arg-before-vararg,too-many-locals limit: Optional[int] = 100, before: Optional[str] = None, after: Optional[str] = None, - *options: ExecutableOption, # pylint: disable=keyword-arg-before-vararg + options: list[ExecutableOption] = None, ) -> List["Node"]: """ Finds a list of nodes by prefix @@ -446,11 +446,11 @@ async def find_by( # pylint: disable=keyword-arg-before-vararg,too-many-locals ) # pragma: no cover if names: statement = statement.where( - Node.name.in_(names), # type: ignore # pylint: disable=no-member + Node.name.in_(names), # type: ignore ) if fragment: statement = statement.where( - Node.name.like(f"%{fragment}%"), # type: ignore # pylint: disable=no-member + Node.name.like(f"%{fragment}%"), # type: ignore ) if node_types: statement = statement.where(Node.type.in_(node_types)) @@ -470,12 +470,12 @@ async def find_by( # pylint: disable=keyword-arg-before-vararg,too-many-locals if after: cursor = NodeCursor.decode(after) statement = statement.where( - (Node.created_at, Node.id) <= (cursor.created_at, cursor.id), # pylint: disable=no-member + (Node.created_at, Node.id) <= (cursor.created_at, cursor.id), ).order_by(Node.created_at.desc(), Node.id.desc()) elif before: cursor = NodeCursor.decode(before) statement = statement.where( - (Node.created_at, Node.id) >= (cursor.created_at, cursor.id), # pylint: disable=no-member + (Node.created_at, Node.id) >= (cursor.created_at, cursor.id), ) statement = statement.order_by(Node.created_at.asc(), Node.id.asc()) else: @@ -483,7 +483,7 @@ async def find_by( # pylint: disable=keyword-arg-before-vararg,too-many-locals limit = limit if limit and limit > 0 else 100 statement = statement.limit(limit) - result = await session.execute(statement.options(*options)) + result = await session.execute(statement.options(*(options or []))) nodes = result.unique().scalars().all() # Reverse for backward pagination @@ -492,7 +492,7 @@ async def find_by( # pylint: disable=keyword-arg-before-vararg,too-many-locals return nodes -class CompressedPickleType(TypeDecorator): # pylint: disable=too-many-ancestors +class CompressedPickleType(TypeDecorator): """ A SQLAlchemy type for storing zlib-compressed pickled objects. """ @@ -536,7 +536,7 @@ def process_literal_param(self, value, dialect): class NodeRevision( Base, -): # pylint: disable=too-few-public-methods,too-many-instance-attributes +): """ A node revision. """ @@ -571,7 +571,7 @@ class NodeRevision( query: Mapped[Optional[str]] = mapped_column(String) mode: Mapped[NodeMode] = mapped_column( Enum(NodeMode), - default=NodeMode.PUBLISHED, # pylint: disable=no-member + default=NodeMode.PUBLISHED, ) version: Mapped[Optional[str]] = mapped_column( @@ -705,7 +705,7 @@ def primary_key(self) -> List[Column]: Returns the primary key columns of this node. """ primary_key_columns = [] - for col in self.columns: # pylint: disable=not-an-iterable + for col in self.columns: if col.has_primary_key_attribute(): primary_key_columns.append(col) return primary_key_columns @@ -715,7 +715,6 @@ def default_load_options(cls): """ Default options when loading a node """ - # pylint: disable=import-outside-toplevel from datajunction_server.database.dimensionlink import DimensionLink return ( @@ -747,12 +746,8 @@ def format_metric_alias(query: str, name: str) -> str: Return a metric query with the metric aliases reassigned to have the same name as the node, if they aren't already matching. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) - from datajunction_server.sql.parsing.backends.antlr4 import ( # pylint: disable=import-outside-toplevel - parse, - ) + from datajunction_server.sql.parsing import ast + from datajunction_server.sql.parsing.backends.antlr4 import parse tree = parse(query) projection_0 = tree.select.projection[0] @@ -768,9 +763,7 @@ def check_metric(self): The Node SQL query should have a single expression in its projections and it should be an aggregation function. """ - from datajunction_server.sql.parsing.backends.antlr4 import ( # pylint: disable=import-outside-toplevel - parse, - ) + from datajunction_server.sql.parsing.backends.antlr4 import parse # must have a single expression tree = parse(self.query) @@ -842,13 +835,13 @@ def copy_dimension_links_from_revision(self, old_revision: "NodeRevision"): Copy dimension links and attributes from another node revision if the column names match """ old_columns_mapping = {col.name: col for col in old_revision.columns} - for col in self.columns: # pylint: disable=not-an-iterable + for col in self.columns: if col.name in old_columns_mapping: col.dimension_id = old_columns_mapping[col.name].dimension_id col.attributes = old_columns_mapping[col.name].attributes or [] return self - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: extra = Extra.allow def has_available_materialization(self, build_criteria: BuildCriteria) -> bool: @@ -857,7 +850,7 @@ def has_available_materialization(self, build_criteria: BuildCriteria) -> bool: """ return ( self.availability is not None # pragma: no cover - and self.availability.is_available( # pylint: disable=no-member + and self.availability.is_available( criteria=build_criteria, ) ) @@ -875,10 +868,7 @@ def cube_elements_with_nodes(self) -> List[Tuple[Column, Optional["NodeRevision" """ Cube elements along with their nodes """ - return [ - (element, element.node_revision()) - for element in self.cube_elements # pylint: disable=not-an-iterable - ] + return [(element, element.node_revision()) for element in self.cube_elements] def cube_metrics(self) -> List[Node]: """ @@ -946,7 +936,7 @@ def temporal_partition_columns(self) -> List[Column]: """ return [ col - for col in self.columns # pylint: disable=not-an-iterable + for col in self.columns if col.partition and col.partition.type_ == PartitionType.TEMPORAL ] @@ -956,7 +946,7 @@ def categorical_partition_columns(self) -> List[Column]: """ return [ col - for col in self.columns # pylint: disable=not-an-iterable + for col in self.columns if col.partition and col.partition.type_ == PartitionType.CATEGORICAL ] @@ -980,18 +970,18 @@ def __deepcopy__(self, memo): return None -class NodeColumns(Base): # pylint: disable=too-few-public-methods +class NodeColumns(Base): """ Join table for node columns. """ __tablename__ = "nodecolumns" - node_id: Mapped[int] = mapped_column( # pylint: disable=unsubscriptable-object + node_id: Mapped[int] = mapped_column( ForeignKey("noderevision.id", name="fk_nodecolumns_node_id_noderevision"), primary_key=True, ) - column_id: Mapped[int] = mapped_column( # pylint: disable=unsubscriptable-object + column_id: Mapped[int] = mapped_column( ForeignKey("column.id", name="fk_nodecolumns_column_id_column"), primary_key=True, ) diff --git a/datajunction-server/datajunction_server/database/partition.py b/datajunction-server/datajunction_server/database/partition.py index 618f405ee..341bc4caf 100644 --- a/datajunction-server/datajunction_server/database/partition.py +++ b/datajunction-server/datajunction_server/database/partition.py @@ -14,7 +14,7 @@ from datajunction_server.sql.parsing.types import TimestampType -class Partition(Base): # type: ignore # pylint: disable=too-few-public-methods +class Partition(Base): # type: ignore """ A partition specification consists of a reference to a partition column and a partition type (either temporal or categorical). Both partition types indicate how to partition the @@ -64,11 +64,7 @@ def temporal_expression(self, interval: Optional[str] = None): CAST(FORMAT(DJ_LOGICAL_TIMESTAMP(), 'yyyyMMdd') AS ). Includes the interval offset in the expression if provided. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) - - # pylint: disable=import-outside-toplevel + from datajunction_server.sql.parsing import ast from datajunction_server.sql.parsing.backends.antlr4 import parse timestamp_expression = ast.Cast( @@ -95,7 +91,7 @@ def temporal_expression(self, interval: Optional[str] = None): ast.String(f"'{self.format}'"), ], ), - data_type=self.column.type, # pylint: disable=no-member + data_type=self.column.type, ) return None # pragma: no cover @@ -103,9 +99,7 @@ def categorical_expression(self): """ Expression for the categorical partition """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast # Register a DJ function (inherits the `datajunction_server.sql.functions.Function` class) # that has the partition column name as the function name. This will be substituted at diff --git a/datajunction-server/datajunction_server/database/queryrequest.py b/datajunction-server/datajunction_server/database/queryrequest.py index 1213c1e34..0e0049e80 100644 --- a/datajunction-server/datajunction_server/database/queryrequest.py +++ b/datajunction-server/datajunction_server/database/queryrequest.py @@ -49,7 +49,7 @@ class QueryBuildType(StrEnum): NODE = "node" -class QueryRequest(Base): # type: ignore # pylint: disable=too-few-public-methods +class QueryRequest(Base): # type: ignore """ A query request represents a request for DJ to build a query. @@ -272,7 +272,7 @@ async def save_query_request( return query_request @classmethod - async def to_versioned_query_request( # pylint: disable=too-many-locals + async def to_versioned_query_request( cls, session: AsyncSession, nodes: List[str], diff --git a/datajunction-server/datajunction_server/database/tag.py b/datajunction-server/datajunction_server/database/tag.py index 50f3daacf..98f64de50 100644 --- a/datajunction-server/datajunction_server/database/tag.py +++ b/datajunction-server/datajunction_server/database/tag.py @@ -16,7 +16,7 @@ from datajunction_server.database.node import Node -class Tag(Base): # pylint: disable=too-few-public-methods +class Tag(Base): """ A tag. """ @@ -102,7 +102,7 @@ async def list_nodes_with_tag( ) -class TagNodeRelationship(Base): # pylint: disable=too-few-public-methods +class TagNodeRelationship(Base): """ Join table between tags and nodes """ diff --git a/datajunction-server/datajunction_server/database/user.py b/datajunction-server/datajunction_server/database/user.py index 932550887..27cde4e14 100644 --- a/datajunction-server/datajunction_server/database/user.py +++ b/datajunction-server/datajunction_server/database/user.py @@ -25,7 +25,7 @@ class OAuthProvider(StrEnum): GOOGLE = "google" -class User(Base): # pylint: disable=too-few-public-methods +class User(Base): """Class for a user.""" __tablename__ = "users" diff --git a/datajunction-server/datajunction_server/enum.py b/datajunction-server/datajunction_server/enum.py index 73198f9bc..cf348c444 100644 --- a/datajunction-server/datajunction_server/enum.py +++ b/datajunction-server/datajunction_server/enum.py @@ -6,7 +6,7 @@ import sys if sys.version_info >= (3, 11): - from enum import ( # noqa # pylint: disable=unused-import # pragma: no cover + from enum import ( # pragma: no cover IntEnum, StrEnum, ) diff --git a/datajunction-server/datajunction_server/errors.py b/datajunction-server/datajunction_server/errors.py index d3dce466e..f8f31f995 100644 --- a/datajunction-server/datajunction_server/errors.py +++ b/datajunction-server/datajunction_server/errors.py @@ -168,7 +168,7 @@ class DJException(Exception): # status code that should be returned when ``DJException`` is caught by the API layer http_status_code: int = 500 - def __init__( # pylint: disable=too-many-arguments + def __init__( self, message: Optional[str] = None, errors: Optional[List[DJError]] = None, diff --git a/datajunction-server/datajunction_server/internal/access/authentication/github.py b/datajunction-server/datajunction_server/internal/access/authentication/github.py index 46e0670d7..df764e986 100644 --- a/datajunction-server/datajunction_server/internal/access/authentication/github.py +++ b/datajunction-server/datajunction_server/internal/access/authentication/github.py @@ -45,7 +45,7 @@ def get_github_user(access_token: str) -> User: # pragma: no cover raise DJAuthenticationException( "Cannot authorize user via GitHub, bad credentials", ) - session = next(get_session()) # type: ignore # pylint: disable=no-value-for-parameter + session = next(get_session()) # type: ignore existing_user = None try: existing_user = session.execute( diff --git a/datajunction-server/datajunction_server/internal/access/authentication/google.py b/datajunction-server/datajunction_server/internal/access/authentication/google.py index 90a21c79f..659007f58 100644 --- a/datajunction-server/datajunction_server/internal/access/authentication/google.py +++ b/datajunction-server/datajunction_server/internal/access/authentication/google.py @@ -22,7 +22,7 @@ _logger = logging.getLogger(__name__) settings = get_settings() -flow = ( # pylint: disable=invalid-name +flow = ( google_auth_oauthlib.flow.Flow.from_client_secrets_file( settings.google_oauth_client_secret_file, scopes=[ @@ -83,7 +83,7 @@ def get_google_user(token: str) -> User: http_status_code=HTTPStatus.FORBIDDEN, message=f"Error retrieving Google user: {response.text}", ) - session = next(get_session()) # type: ignore # pylint: disable=no-value-for-parameter + session = next(get_session()) # type: ignore existing_user = session.execute( select(User).where(User.email == user_data["login"]), ).scalar() diff --git a/datajunction-server/datajunction_server/internal/access/authentication/http.py b/datajunction-server/datajunction_server/internal/access/authentication/http.py index c0a25b430..bb3564379 100644 --- a/datajunction-server/datajunction_server/internal/access/authentication/http.py +++ b/datajunction-server/datajunction_server/internal/access/authentication/http.py @@ -20,7 +20,7 @@ from datajunction_server.utils import get_session, get_settings -class DJHTTPBearer(HTTPBearer): # pylint: disable=too-few-public-methods +class DJHTTPBearer(HTTPBearer): """ A custom HTTPBearer that accepts a cookie or bearer token """ diff --git a/datajunction-server/datajunction_server/internal/access/authorization.py b/datajunction-server/datajunction_server/internal/access/authorization.py index 303fb05bc..8fd26aa71 100644 --- a/datajunction-server/datajunction_server/internal/access/authorization.py +++ b/datajunction-server/datajunction_server/internal/access/authorization.py @@ -16,7 +16,7 @@ def validate_access_requests( - validate_access: ValidateAccessFn, # pylint: disable=W0621 + validate_access: ValidateAccessFn, user: User, resource_requests: Iterable[ResourceRequest], raise_: bool = False, diff --git a/datajunction-server/datajunction_server/internal/cube_materializations.py b/datajunction-server/datajunction_server/internal/cube_materializations.py index 60ece4945..3bd17589f 100644 --- a/datajunction-server/datajunction_server/internal/cube_materializations.py +++ b/datajunction-server/datajunction_server/internal/cube_materializations.py @@ -137,7 +137,7 @@ def _combine_measures_join_criteria(left_table, right_table, query_grain): ) -async def build_cube_materialization( # pylint: disable=used-before-assignment,too-many-locals +async def build_cube_materialization( session: AsyncSession, current_revision: NodeRevision, upsert_input: UpsertCubeMaterialization, @@ -267,6 +267,6 @@ async def build_cube_materialization( # pylint: disable=used-before-assignment, ], dimensions=current_revision.cube_node_dimensions, measures_materializations=measures_materializations, - combiners=combiners, # pylint: disable=possibly-used-before-assignment + combiners=combiners, ) return config diff --git a/datajunction-server/datajunction_server/internal/materializations.py b/datajunction-server/datajunction_server/internal/materializations.py index d9e3eb74a..c158b32ad 100644 --- a/datajunction-server/datajunction_server/internal/materializations.py +++ b/datajunction-server/datajunction_server/internal/materializations.py @@ -21,6 +21,7 @@ from datajunction_server.materialization.jobs import MaterializationJob from datajunction_server.models import access from datajunction_server.models.column import SemanticType +from datajunction_server.models.cube_materialization import UpsertCubeMaterialization from datajunction_server.models.materialization import ( DruidMeasuresCubeConfig, DruidMetricsCubeConfig, @@ -58,7 +59,7 @@ async def rewrite_metrics_expressions( metrics_expressions = {} measures_to_output_columns_lookup = { column.semantic_entity: column.name - for column in measures_query.columns # type: ignore # pylint: disable=not-an-iterable + for column in measures_query.columns # type: ignore } for metric in current_revision.cube_metrics(): measures_for_metric = [] @@ -80,7 +81,7 @@ async def rewrite_metrics_expressions( ), ) - col._table = None # pylint: disable=protected-access + col._table = None col.name = ast.Name( measures_to_output_columns_lookup[full_column_name], ) @@ -134,12 +135,12 @@ async def build_cube_materialization_config( query=metrics_query.sql, dimensions=[ col.name - for col in metrics_query.columns # type: ignore # pylint: disable=not-an-iterable + for col in metrics_query.columns # type: ignore if col.semantic_type != SemanticType.METRIC ], metrics=[ col - for col in metrics_query.columns # type: ignore # pylint: disable=not-an-iterable + for col in metrics_query.columns # type: ignore if col.semantic_type == SemanticType.METRIC ], spark=upsert_input.config.spark, @@ -168,7 +169,7 @@ async def build_cube_materialization_config( query=measures_query.sql, dimensions=[ col.name - for col in measures_query.columns # type: ignore # pylint: disable=not-an-iterable + for col in measures_query.columns # type: ignore if col.semantic_type == SemanticType.DIMENSION ], measures=metrics_expressions, @@ -228,7 +229,7 @@ async def build_non_cube_materialization_config( async def create_new_materialization( session: AsyncSession, current_revision: NodeRevision, - upsert: UpsertMaterialization, + upsert: UpsertMaterialization | UpsertCubeMaterialization, validate_access: access.ValidateAccessFn, current_user: User, ) -> Materialization: @@ -341,7 +342,7 @@ def _get_readable_name(expr): ) -def decompose_expression( # pylint: disable=too-many-return-statements +def decompose_expression( expr: Union[ast.Aliasable, ast.Expression], ) -> Tuple[ast.Expression, List[ast.Alias]]: """ diff --git a/datajunction-server/datajunction_server/internal/namespaces.py b/datajunction-server/datajunction_server/internal/namespaces.py index 3adcb2938..08e24f2f8 100644 --- a/datajunction-server/datajunction_server/internal/namespaces.py +++ b/datajunction-server/datajunction_server/internal/namespaces.py @@ -68,7 +68,7 @@ async def get_nodes_in_namespace_detailed( select(Node) .where( or_( - Node.namespace.like(f"{namespace}.%"), # pylint: disable=no-member + Node.namespace.like(f"{namespace}.%"), Node.namespace == namespace, ), Node.current_version == NodeRevision.version, @@ -85,7 +85,7 @@ async def get_nodes_in_namespace_detailed( return (await session.execute(list_nodes_query)).unique().scalars().all() -async def list_namespaces_in_hierarchy( # pylint: disable=too-many-arguments +async def list_namespaces_in_hierarchy( session: AsyncSession, namespace: str, ) -> List[NodeNamespace]: @@ -94,7 +94,7 @@ async def list_namespaces_in_hierarchy( # pylint: disable=too-many-arguments """ statement = select(NodeNamespace).where( or_( - NodeNamespace.namespace.like( # pylint: disable=no-member + NodeNamespace.namespace.like( f"{namespace}.%", ), NodeNamespace.namespace == namespace, @@ -240,7 +240,7 @@ async def hard_delete_namespace( or_( Node.namespace.like( f"{namespace}.%", - ), # pylint: disable=no-member + ), Node.namespace == namespace, ), ) diff --git a/datajunction-server/datajunction_server/internal/nodes.py b/datajunction-server/datajunction_server/internal/nodes.py index 72009dfd2..a1103f2fa 100644 --- a/datajunction-server/datajunction_server/internal/nodes.py +++ b/datajunction-server/datajunction_server/internal/nodes.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines,too-many-arguments """Nodes endpoint helper functions""" import logging @@ -63,6 +62,7 @@ MaterializationJobTypeEnum, UpsertMaterialization, ) +from datajunction_server.models.cube_materialization import UpsertCubeMaterialization from datajunction_server.models.node import ( DEFAULT_DRAFT_VERSION, DEFAULT_PUBLISHED_VERSION, @@ -278,7 +278,6 @@ async def create_node_revision( ( await session.execute( select(Node).where( - # pylint: disable=no-member Node.name.in_( # type: ignore new_parents, ), @@ -305,7 +304,7 @@ async def create_node_revision( return node_revision -async def create_cube_node_revision( # pylint: disable=too-many-locals +async def create_cube_node_revision( session: AsyncSession, data: CreateCubeNode, current_user: User, @@ -418,7 +417,7 @@ async def save_node( await propagate_valid_status( session=session, valid_nodes=newly_valid_nodes, - catalog_id=node.current.catalog_id, # pylint: disable=no-member + catalog_id=node.current.catalog_id, current_user=current_user, ) await session.refresh(node.current) @@ -479,7 +478,7 @@ async def copy_to_new_node( for missing_parent in old_revision.missing_parents ], columns=[col.copy() for col in old_revision.columns], - # TODO: availability and materializations are missing here # pylint: disable=fixme + # TODO: availability and materializations are missing here lineage=old_revision.lineage, created_by_id=current_user.id, custom_metadata=old_revision.custom_metadata, @@ -539,7 +538,7 @@ async def copy_to_new_node( await propagate_valid_status( session=session, valid_nodes=newly_valid_nodes, - catalog_id=node.current.catalog_id, # type: ignore # pylint: disable=no-member + catalog_id=node.current.catalog_id, # type: ignore current_user=current_user, ) await session.refresh(node.current) # type: ignore @@ -666,16 +665,25 @@ async def update_node_with_query( await session.refresh(new_revision, ["materializations"]) if active_materializations and new_revision.query != old_revision.query: for old in active_materializations: - new_revision.materializations.append( # pylint: disable=no-member + new_revision.materializations.append( await create_new_materialization( session, new_revision, - UpsertMaterialization( + UpsertMaterialization( # type: ignore name=old.name, config=old.config, schedule=old.schedule, strategy=old.strategy, job=MaterializationJobTypeEnum.find_match(old.job), + ) + if old.job != MaterializationJobTypeEnum.DRUID_CUBE.value.job_class + else ( + UpsertCubeMaterialization( + job=MaterializationJobTypeEnum.find_match(old.job), + strategy=old.strategy, + schedule=old.schedule, + lookback_window=old.lookback_window, + ) ), validate_access, current_user=current_user, @@ -701,7 +709,7 @@ async def update_node_with_query( session=session, node_revision=new_revision, ) - # TODO: Do not save this until: # pylint: disable=fixme + # TODO: Do not save this until: # 1. We get to the bottom of why there are query building discrepancies # 2. We audit our database calls to defer pulling the query_ast in most cases # background_tasks.add_task( @@ -771,7 +779,7 @@ def node_update_history_event(new_revision: NodeRevision, current_user: User): ) -async def update_cube_node( # pylint: disable=too-many-locals +async def update_cube_node( session: AsyncSession, node_revision: NodeRevision, data: UpdateNode, @@ -843,7 +851,7 @@ async def update_cube_node( # pylint: disable=too-many-locals ] if major_changes and active_materializations: for old in active_materializations: - new_cube_revision.materializations.append( # pylint: disable=no-member + new_cube_revision.materializations.append( await create_new_materialization( session, new_cube_revision, @@ -900,7 +908,7 @@ async def update_cube_node( # pylint: disable=too-many-locals return new_cube_revision -async def propagate_update_downstream( # pylint: disable=too-many-locals +async def propagate_update_downstream( session: AsyncSession, node: Node, *, @@ -1009,7 +1017,7 @@ def copy_existing_node_revision(old_revision: NodeRevision, current_user: User): ) -async def create_node_from_inactive( # pylint: disable=too-many-arguments +async def create_node_from_inactive( new_node_type: NodeType, data: Union[CreateSourceNode, CreateNode, CreateCubeNode], session: AsyncSession, @@ -1036,7 +1044,7 @@ async def create_node_from_inactive( # pylint: disable=too-many-arguments if previous_inactive_node and previous_inactive_node.deactivated_at: if previous_inactive_node.type != new_node_type: raise DJInvalidInputException( # pragma: no cover - message=f"A node with name `{data.name}` of a `{previous_inactive_node.type.value}` " # pylint: disable=line-too-long + message=f"A node with name `{data.name}` of a `{previous_inactive_node.type.value}` " "type existed before. If you want to re-create it with a different type, " "you need to remove all traces of the previous node with a hard delete call: " "DELETE /nodes/{node_name}/hard", @@ -1095,7 +1103,7 @@ async def create_node_from_inactive( # pylint: disable=too-many-arguments return None -async def create_new_revision_from_existing( # pylint: disable=too-many-locals,too-many-branches,too-many-statements +async def create_new_revision_from_existing( session: AsyncSession, old_revision: NodeRevision, node: Node, @@ -1250,7 +1258,6 @@ async def create_new_revision_from_existing( # pylint: disable=too-many-locals, await session.execute( select(Node) .where( - # pylint: disable=no-member Node.name.in_( # type: ignore new_parents, ), @@ -1938,7 +1945,7 @@ async def activate_node( await session.commit() -async def revalidate_node( # pylint: disable=too-many-locals,too-many-statements,too-many-branches +async def revalidate_node( name: str, session: AsyncSession, current_user: User, diff --git a/datajunction-server/datajunction_server/internal/validation.py b/datajunction-server/datajunction_server/internal/validation.py index f4e4f986e..3ac6cc5ef 100644 --- a/datajunction-server/datajunction_server/internal/validation.py +++ b/datajunction-server/datajunction_server/internal/validation.py @@ -19,7 +19,7 @@ @dataclass -class NodeValidator: # pylint: disable=too-many-instance-attributes +class NodeValidator: """ Node validation """ @@ -50,7 +50,7 @@ def modified_columns(self, node_revision: NodeRevision) -> Set[str]: return updated_columns -async def validate_node_data( # pylint: disable=too-many-locals,too-many-statements +async def validate_node_data( data: Union[NodeRevisionBase, NodeRevision], session: AsyncSession, ) -> NodeValidator: diff --git a/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py b/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py index 866a032c7..58affe56a 100644 --- a/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py +++ b/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py @@ -31,9 +31,7 @@ _logger = logging.getLogger(__name__) -class DefaultCubeMaterialization( - MaterializationJob, -): # pylint: disable=too-few-public-methods +class DefaultCubeMaterialization(MaterializationJob): """ Dummy job that is not meant to be executed but contains all the settings needed for to materialize a generic cube. diff --git a/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py b/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py index 35d9b90c3..31e780fff 100644 --- a/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py +++ b/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py @@ -23,7 +23,7 @@ settings = get_settings() -class MaterializationJob(abc.ABC): # pylint: disable=too-few-public-methods +class MaterializationJob(abc.ABC): """ Base class for a materialization job """ @@ -63,7 +63,7 @@ def schedule( """ -class SparkSqlMaterializationJob( # pylint: disable=too-few-public-methods # pragma: no cover +class SparkSqlMaterializationJob( # pragma: no cover MaterializationJob, ): """ diff --git a/datajunction-server/datajunction_server/models/attribute.py b/datajunction-server/datajunction_server/models/attribute.py index 1694433bc..04993d556 100644 --- a/datajunction-server/datajunction_server/models/attribute.py +++ b/datajunction-server/datajunction_server/models/attribute.py @@ -46,7 +46,7 @@ class AttributeTypeBase(MutableAttributeTypeFields): id: int - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/catalog.py b/datajunction-server/datajunction_server/models/catalog.py index 71d47ac20..633a16553 100644 --- a/datajunction-server/datajunction_server/models/catalog.py +++ b/datajunction-server/datajunction_server/models/catalog.py @@ -20,5 +20,5 @@ class CatalogInfo(BaseModel): name: str engines: Optional[List[EngineInfo]] = [] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/collection.py b/datajunction-server/datajunction_server/models/collection.py index f4b9ba8bf..99be447d8 100644 --- a/datajunction-server/datajunction_server/models/collection.py +++ b/datajunction-server/datajunction_server/models/collection.py @@ -18,7 +18,7 @@ class CollectionInfo(BaseModel): name: str description: str - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -32,5 +32,5 @@ class CollectionDetails(CollectionInfo): description: str nodes: list[NodeNameOutput] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/column.py b/datajunction-server/datajunction_server/models/column.py index 4e2ed9818..89d6c486e 100644 --- a/datajunction-server/datajunction_server/models/column.py +++ b/datajunction-server/datajunction_server/models/column.py @@ -21,9 +21,7 @@ class ColumnYAML(TypedDict, total=False): dimension: str -class ColumnTypeDecorator( - TypeDecorator, -): # pylint: disable=abstract-method, too-many-ancestors +class ColumnTypeDecorator(TypeDecorator): """ Converts a column type from the database to a `ColumnType` class """ @@ -35,9 +33,7 @@ def process_bind_param(self, value: ColumnType, dialect): return str(value) def process_result_value(self, value, dialect): - from datajunction_server.sql.parsing.backends.antlr4 import ( # pylint: disable=import-outside-toplevel - parse_rule, - ) + from datajunction_server.sql.parsing.backends.antlr4 import parse_rule if not value: return value diff --git a/datajunction-server/datajunction_server/models/cube.py b/datajunction-server/datajunction_server/models/cube.py index e02761598..4c478fd79 100644 --- a/datajunction-server/datajunction_server/models/cube.py +++ b/datajunction-server/datajunction_server/models/cube.py @@ -32,7 +32,7 @@ class CubeElementMetadata(BaseModel): partition: Optional[PartitionOutput] @root_validator(pre=True) - def type_string(cls, values): # pylint: disable=no-self-argument + def type_string(cls, values): """ Extracts the type as a string """ @@ -46,7 +46,7 @@ def type_string(cls, values): # pylint: disable=no-self-argument ) return values - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: orm_mode = True @@ -75,7 +75,7 @@ class CubeRevisionMetadata(BaseModel): materializations: List[MaterializationConfigOutput] tags: Optional[List[TagOutput]] - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: allow_population_by_field_name = True orm_mode = True diff --git a/datajunction-server/datajunction_server/models/cube_materialization.py b/datajunction-server/datajunction_server/models/cube_materialization.py index 0c9b011f7..d945b03a7 100644 --- a/datajunction-server/datajunction_server/models/cube_materialization.py +++ b/datajunction-server/datajunction_server/models/cube_materialization.py @@ -125,9 +125,7 @@ def table_ast(self): """ Generate a unique output table name based on the parameters. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast return ast.Table(name=ast.Name(self.output_table_name)) @@ -174,7 +172,7 @@ def from_measures_query(cls, measures_query, temporal_partition): query=measures_query.sql, dimensions=[ col.name - for col in measures_query.columns # type: ignore # pylint: disable=not-an-iterable + for col in measures_query.columns # type: ignore if col.semantic_type == SemanticType.DIMENSION ], measures=list( @@ -210,7 +208,7 @@ class CubeMetric(BaseModel): derived_expression: str = Field( description=( "The expression for rewriting the original metric query " - "using the materialized measures.", + "using the materialized measures." ), ) @@ -233,7 +231,7 @@ class UpsertCubeMaterialization(BaseModel): lookback_window: str | None = "1 DAY" @validator("job", pre=True) - def validate_job( # pylint: disable=no-self-argument + def validate_job( cls, job: Union[str, MaterializationJobTypeEnum], ) -> MaterializationJobTypeEnum: @@ -242,9 +240,7 @@ def validate_job( # pylint: disable=no-self-argument """ if isinstance(job, str): # pragma: no cover job_name = job.upper() - options = ( - MaterializationJobTypeEnum._member_names_ # pylint: disable=protected-access,no-member - ) + options = MaterializationJobTypeEnum._member_names_ if job_name not in options: raise DJInvalidInputException( http_status_code=404, @@ -344,7 +340,6 @@ def build_druid_spec(self): "format": "parquet", "dimensionsSpec": { "dimensions": sorted( - # pylint: disable=no-member list(set(self.dimensions)), # type: ignore ), }, diff --git a/datajunction-server/datajunction_server/models/dimensionlink.py b/datajunction-server/datajunction_server/models/dimensionlink.py index 47b831d21..8b443cfab 100644 --- a/datajunction-server/datajunction_server/models/dimensionlink.py +++ b/datajunction-server/datajunction_server/models/dimensionlink.py @@ -64,5 +64,5 @@ class LinkDimensionOutput(BaseModel): role: Optional[str] foreign_keys: Dict[str, str] - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/engine.py b/datajunction-server/datajunction_server/models/engine.py index c23201259..5465df8f0 100644 --- a/datajunction-server/datajunction_server/models/engine.py +++ b/datajunction-server/datajunction_server/models/engine.py @@ -29,7 +29,7 @@ class EngineInfo(BaseModel): uri: Optional[str] dialect: Optional[Dialect] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/history.py b/datajunction-server/datajunction_server/models/history.py index 8805ae9c2..6f11d9d7f 100644 --- a/datajunction-server/datajunction_server/models/history.py +++ b/datajunction-server/datajunction_server/models/history.py @@ -31,7 +31,7 @@ class HistoryOutput(BaseModel): details: Dict[str, Any] created_at: UTCDatetime - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/materialization.py b/datajunction-server/datajunction_server/models/materialization.py index 5c727e16d..e50ba07e9 100644 --- a/datajunction-server/datajunction_server/models/materialization.py +++ b/datajunction-server/datajunction_server/models/materialization.py @@ -126,7 +126,7 @@ class MaterializationConfigOutput(BaseModel): backfills: List[BackfillOutput] strategy: Optional[str] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -290,7 +290,7 @@ def metrics_spec(self) -> Dict: """ Returns the Druid metrics spec for ingestion """ - self.dimensions += [ # type: ignore # pylint: disable=no-member + self.dimensions += [ # type: ignore measure.field_name for measure_group in self.measures.values() # type: ignore for measure in measure_group.measures @@ -346,7 +346,6 @@ def build_druid_spec(self, node_revision: "NodeRevision"): "format": "parquet", "dimensionsSpec": { "dimensions": sorted( - # pylint: disable=no-member list(set(self.dimensions)), # type: ignore ), }, @@ -383,9 +382,7 @@ def build_druid_spec(self, node_revision: "NodeRevision"): return druid_spec -class DruidMetricsCubeConfig( - DruidMeasuresCubeConfig, -): # pylint: disable=too-many-ancestors +class DruidMetricsCubeConfig(DruidMeasuresCubeConfig): """ Specific cube materialization implementation with Spark and Druid ingestion and optional prefix and/or suffix to include with the materialized entity's name. @@ -494,7 +491,7 @@ class UpsertMaterialization(BaseModel): strategy: MaterializationStrategy @validator("job", pre=True) - def validate_job( # pylint: disable=no-self-argument + def validate_job( cls, job: Union[str, MaterializationJobTypeEnum], ) -> MaterializationJobTypeEnum: @@ -503,9 +500,7 @@ def validate_job( # pylint: disable=no-self-argument """ if isinstance(job, str): job_name = job.upper() - options = ( - MaterializationJobTypeEnum._member_names_ # pylint: disable=protected-access,no-member - ) + options = MaterializationJobTypeEnum._member_names_ if job_name not in options: raise DJInvalidInputException( http_status_code=404, diff --git a/datajunction-server/datajunction_server/models/measure.py b/datajunction-server/datajunction_server/models/measure.py index 58adec3c1..fa807a71f 100644 --- a/datajunction-server/datajunction_server/models/measure.py +++ b/datajunction-server/datajunction_server/models/measure.py @@ -65,7 +65,7 @@ class ColumnOutput(BaseModel): node: str @root_validator(pre=True) - def transform(cls, values): # pylint: disable=no-self-argument + def transform(cls, values): """ Transforms the values for output """ @@ -75,7 +75,7 @@ def transform(cls, values): # pylint: disable=no-self-argument "node": values.get("node_revisions")[0].name, } - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -90,5 +90,5 @@ class MeasureOutput(BaseModel): columns: List[ColumnOutput] additive: AggregationRule - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/metric.py b/datajunction-server/datajunction_server/models/metric.py index 999ceca7a..aaeb35d3d 100644 --- a/datajunction-server/datajunction_server/models/metric.py +++ b/datajunction-server/datajunction_server/models/metric.py @@ -90,7 +90,7 @@ class TranslatedSQL(BaseModel): Class for SQL generated from a given metric. """ - # TODO: once type-inference is added to /query/ endpoint # pylint: disable=fixme + # TODO: once type-inference is added to /query/ endpoint # columns attribute can be required sql: str columns: Optional[List[ColumnMetadata]] = None # pragma: no-cover @@ -98,7 +98,7 @@ class TranslatedSQL(BaseModel): upstream_tables: Optional[List[str]] = None @root_validator(pre=False) - def transpile_sql( # pylint: disable=no-self-argument + def transpile_sql( cls, values, ) -> "TranslatedSQL": diff --git a/datajunction-server/datajunction_server/models/node.py b/datajunction-server/datajunction_server/models/node.py index a624111ce..f03bc779b 100644 --- a/datajunction-server/datajunction_server/models/node.py +++ b/datajunction-server/datajunction_server/models/node.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-instance-attributes,too-many-lines,too-many-ancestors """ Model for nodes. """ @@ -283,11 +282,11 @@ class AvailabilityStateBase(TemporalPartitionRange): # Partition-level availabilities partitions: Optional[List[PartitionAvailability]] = Field(default=[]) - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @validator("partitions") - def validate_partitions(cls, partitions): # pylint: disable=no-self-argument + def validate_partitions(cls, partitions): """ Validator for partitions """ @@ -381,7 +380,7 @@ def __repr__(self): return self.name @validator("label", always=True) - def get_label( # pylint: disable=no-self-argument + def get_label( cls, label: str, values: Dict[str, Any], @@ -391,14 +390,14 @@ def get_label( # pylint: disable=no-self-argument return labelize(values["name"]) return label - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True class MetricUnit(enum.Enum): """ Available units of measure for metrics - TODO: Eventually this can be recorded in a database, # pylint: disable=fixme + TODO: Eventually this can be recorded in a database, since measurement units can be customized depending on the metric (i.e., clicks/hour). For the time being, this enum provides some basic units. """ @@ -465,7 +464,7 @@ class MetricMetadataOutput(BaseModel): direction: Optional[MetricDirection] unit: Optional[Unit] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -542,7 +541,7 @@ class NodeMinimumDetail(BaseModel): tags: Optional[List[TagMinimum]] edited_by: Optional[List[str]] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -554,7 +553,7 @@ class AttributeTypeName(BaseModel): namespace: str name: str - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -565,7 +564,7 @@ class AttributeOutput(BaseModel): attribute_type: AttributeTypeName - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -595,7 +594,7 @@ class ColumnOutput(BaseModel): dimension: Optional[NodeNameOutput] partition: Optional[PartitionOutput] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: """ Should perform validation on assignment """ @@ -604,7 +603,7 @@ class Config: # pylint: disable=missing-class-docstring, too-few-public-methods validate_assignment = True _extract_type = validator("type", pre=True, allow_reuse=True)( - lambda raw: str(raw), # pylint: disable=unnecessary-lambda + lambda raw: str(raw), ) @@ -618,7 +617,7 @@ class SourceColumnOutput(BaseModel): attributes: Optional[List[AttributeOutput]] dimension: Optional[str] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: """ Should perform validation on assignment """ @@ -626,7 +625,7 @@ class Config: # pylint: disable=missing-class-docstring, too-few-public-methods validate_assignment = True @root_validator - def type_string(cls, values): # pylint: disable=no-self-argument + def type_string(cls, values): """ Extracts the type as a string """ @@ -713,14 +712,14 @@ class UpdateNode( __annotations__ = { k: Optional[v] for k, v in { - **SourceNodeFields.__annotations__, # pylint: disable=E1101 - **MutableNodeFields.__annotations__, # pylint: disable=E1101 - **MutableNodeQueryField.__annotations__, # pylint: disable=E1101 - **CubeNodeFields.__annotations__, # pylint: disable=E1101 + **SourceNodeFields.__annotations__, + **MutableNodeFields.__annotations__, + **MutableNodeQueryField.__annotations__, + **CubeNodeFields.__annotations__, }.items() } - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: """ Do not allow fields other than the ones defined here. """ @@ -740,7 +739,7 @@ class GenericNodeOutputModel(BaseModel): """ @root_validator(pre=True) - def flatten_current( # pylint: disable=no-self-argument + def flatten_current( cls, values: GetterDict, ) -> Union[GetterDict, Dict[str, Any]]: @@ -806,7 +805,7 @@ class NodeRevisionOutput(BaseModel): dimension_links: Optional[List[LinkDimensionOutput]] custom_metadata: Optional[Dict] = None - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: orm_mode = True @@ -843,7 +842,7 @@ class NodeOutput(GenericNodeOutputModel): missing_table: Optional[bool] = False custom_metadata: Optional[Dict] = None - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: orm_mode = True @classmethod @@ -851,7 +850,7 @@ def load_options(cls): """ ORM options to successfully load this object """ - from datajunction_server.database.node import ( # pylint: disable=import-outside-toplevel + from datajunction_server.database.node import ( Node, NodeRevision, ) @@ -885,7 +884,7 @@ class DAGNodeRevisionOutput(BaseModel): parents: List[NodeNameOutput] dimension_links: List[LinkDimensionOutput] - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: allow_population_by_field_name = True orm_mode = True @@ -916,7 +915,7 @@ class DAGNodeOutput(GenericNodeOutputModel): tags: List[TagOutput] = [] current_version: str - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/node_type.py b/datajunction-server/datajunction_server/models/node_type.py index 798454d37..9f91fab5b 100644 --- a/datajunction-server/datajunction_server/models/node_type.py +++ b/datajunction-server/datajunction_server/models/node_type.py @@ -32,7 +32,7 @@ class NodeNameOutput(BaseModel): name: str - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -44,5 +44,5 @@ class NodeNameVersion(BaseModel): name: str version: str - class Config: # pylint: disable=missing-class-docstring,too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/partition.py b/datajunction-server/datajunction_server/models/partition.py index 567c83859..9a088935e 100644 --- a/datajunction-server/datajunction_server/models/partition.py +++ b/datajunction-server/datajunction_server/models/partition.py @@ -66,7 +66,7 @@ class PartitionBackfill(BaseModel): values: Optional[List] range: Optional[List] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -80,7 +80,7 @@ class PartitionOutput(BaseModel): granularity: Optional[str] expression: Optional[str] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -103,5 +103,5 @@ class BackfillOutput(BaseModel): spec: Optional[List[PartitionBackfill]] urls: Optional[List[str]] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/models/query.py b/datajunction-server/datajunction_server/models/query.py index 749d96ba5..bdb5968d7 100644 --- a/datajunction-server/datajunction_server/models/query.py +++ b/datajunction-server/datajunction_server/models/query.py @@ -23,7 +23,7 @@ class BaseQuery(BaseModel): engine_name: Optional[str] = None engine_version: Optional[str] = None - class Config: # pylint: disable=too-few-public-methods, missing-class-docstring + class Config: allow_population_by_field_name = True @@ -114,21 +114,21 @@ class QueryWithResults(BaseModel): links: Optional[List[AnyHttpUrl]] = None @validator("scheduled", pre=True) - def parse_scheduled_date_string(cls, value): # pylint: disable=no-self-argument + def parse_scheduled_date_string(cls, value): """ Convert string date values to datetime """ return datetime.fromisoformat(value) if isinstance(value, str) else value @validator("started", pre=True) - def parse_started_date_string(cls, value): # pylint: disable=no-self-argument + def parse_started_date_string(cls, value): """ Convert string date values to datetime """ return datetime.fromisoformat(value) if isinstance(value, str) else value @validator("finished", pre=True) - def parse_finisheddate_string(cls, value): # pylint: disable=no-self-argument + def parse_finisheddate_string(cls, value): """ Convert string date values to datetime """ diff --git a/datajunction-server/datajunction_server/models/sql.py b/datajunction-server/datajunction_server/models/sql.py index 4a76a4de7..8cd0c388b 100644 --- a/datajunction-server/datajunction_server/models/sql.py +++ b/datajunction-server/datajunction_server/models/sql.py @@ -32,7 +32,7 @@ class GeneratedSQL(BaseModel): errors: Optional[List[DJQueryBuildError]] = None @root_validator(pre=False) - def transpile_sql( # pylint: disable=no-self-argument + def transpile_sql( cls, values, ): diff --git a/datajunction-server/datajunction_server/models/table.py b/datajunction-server/datajunction_server/models/table.py index c9a49a2a2..54d63830f 100644 --- a/datajunction-server/datajunction_server/models/table.py +++ b/datajunction-server/datajunction_server/models/table.py @@ -19,7 +19,7 @@ class TableYAML(TypedDict, total=False): cost: float -class TableBase(BaseModel): # pylint: disable=too-few-public-methods +class TableBase(BaseModel): """ A base table. """ diff --git a/datajunction-server/datajunction_server/models/tag.py b/datajunction-server/datajunction_server/models/tag.py index f9ab5ff08..5163420d5 100644 --- a/datajunction-server/datajunction_server/models/tag.py +++ b/datajunction-server/datajunction_server/models/tag.py @@ -20,7 +20,7 @@ class MutableTagFields(BaseModel): display_name: Optional[str] tag_metadata: Optional[Dict[str, Any]] = {} - class Config: # pylint: disable=too-few-public-methods + class Config: """ Allow types for tag metadata. """ @@ -50,7 +50,7 @@ class TagMinimum(BaseModel): name: str - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -59,7 +59,7 @@ class TagOutput(ImmutableTagFields, MutableTagFields): Output tag model. """ - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -71,11 +71,11 @@ class UpdateTag(MutableTagFields): __annotations__ = { k: Optional[v] for k, v in { - **MutableTagFields.__annotations__, # pylint: disable=E1101 + **MutableTagFields.__annotations__, }.items() } - class Config: # pylint: disable=too-few-public-methods + class Config: """ Do not allow fields other than the ones defined here. """ diff --git a/datajunction-server/datajunction_server/models/user.py b/datajunction-server/datajunction_server/models/user.py index b0337911e..404bcb038 100644 --- a/datajunction-server/datajunction_server/models/user.py +++ b/datajunction-server/datajunction_server/models/user.py @@ -31,7 +31,7 @@ class CreatedNode(BaseModel): current_version: str missing_table: Optional[bool] = False - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -48,7 +48,7 @@ class UserOutput(BaseModel): created_nodes: Optional[List[CreatedNode]] = [] created_tags: Optional[List[TagOutput]] = [] - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True @@ -59,7 +59,7 @@ class UserNameOnly(BaseModel): username: str - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: orm_mode = True diff --git a/datajunction-server/datajunction_server/service_clients.py b/datajunction-server/datajunction_server/service_clients.py index 064c0d60b..86cc8cba6 100644 --- a/datajunction-server/datajunction_server/service_clients.py +++ b/datajunction-server/datajunction_server/service_clients.py @@ -73,7 +73,7 @@ def construct_url(self, url): return urljoin(self.endpoint, url) -class QueryServiceClient: # pylint: disable=too-few-public-methods +class QueryServiceClient: """ Client for the query service. """ @@ -148,7 +148,7 @@ def get_columns_for_table( for idx, column in enumerate(table_columns) ] - def create_view( # pylint: disable=too-many-arguments + def create_view( self, view_name: str, query_create: QueryCreate, @@ -179,7 +179,7 @@ def create_view( # pylint: disable=too-many-arguments ) return f"View '{view_name}' created successfully." - def submit_query( # pylint: disable=too-many-arguments + def submit_query( self, query_create: QueryCreate, request_headers: Optional[Dict[str, str]] = None, diff --git a/datajunction-server/datajunction_server/sql/dag.py b/datajunction-server/datajunction_server/sql/dag.py index 579b63065..b47a85924 100644 --- a/datajunction-server/datajunction_server/sql/dag.py +++ b/datajunction-server/datajunction_server/sql/dag.py @@ -111,7 +111,7 @@ async def get_downstream_nodes( max_depths = ( select( paths.c.node_id, - func.max(paths.c.depth).label("max_depth"), # pylint: disable=not-callable + func.max(paths.c.depth).label("max_depth"), ) .group_by(paths.c.node_id) .cte("max_depths") @@ -226,7 +226,7 @@ async def get_upstream_nodes( ] -async def get_dimensions_dag( # pylint: disable=too-many-locals +async def get_dimensions_dag( session: AsyncSession, node_revision: NodeRevision, with_attributes: bool = True, @@ -267,7 +267,7 @@ async def get_dimensions_dag( # pylint: disable=too-many-locals DimensionLink.dimension_id, ( literal("[") - + func.coalesce( # pylint: disable=not-callable + + func.coalesce( DimensionLink.role, literal(""), ) @@ -696,18 +696,18 @@ async def get_nodes_with_dimension( onclause=( (NodeRevision.node_id == Node.id) & (Node.current_version == NodeRevision.version) - ), # pylint: disable=superfluous-parens + ), ) .join( NodeColumns, - onclause=(NodeRevision.id == NodeColumns.node_id), # pylint: disable=superfluous-parens + onclause=(NodeRevision.id == NodeColumns.node_id), ) .join( Column, - onclause=(NodeColumns.column_id == Column.id), # pylint: disable=superfluous-parens + onclause=(NodeColumns.column_id == Column.id), ) .where( - Column.dimension_id.in_( # type: ignore # pylint: disable=no-member + Column.dimension_id.in_( # type: ignore [current_node.id], ), ) @@ -735,7 +735,7 @@ async def get_nodes_with_dimension( onclause=( (NodeRevision.node_id == Node.id) & (Node.current_version == NodeRevision.version) - ), # pylint: disable=superfluous-parens + ), ) .where(DimensionLink.dimension_id.in_([current_node.id])) ) @@ -852,7 +852,7 @@ async def get_dimension_dag_indegree(session, node_names: List[str]) -> Dict[str statement = ( select( DimensionLink.dimension_id, - func.count(DimensionLink.id), # pylint: disable=not-callable + func.count(DimensionLink.id), ) .where(DimensionLink.dimension_id.in_(dimension_ids)) .group_by(DimensionLink.dimension_id) diff --git a/datajunction-server/datajunction_server/sql/functions.py b/datajunction-server/datajunction_server/sql/functions.py index 11852f0b1..c16593347 100644 --- a/datajunction-server/datajunction_server/sql/functions.py +++ b/datajunction-server/datajunction_server/sql/functions.py @@ -1,5 +1,3 @@ -# pylint: disable=too-many-lines, abstract-method, unused-argument, missing-function-docstring, -# pylint: disable=arguments-differ, too-many-return-statements, function-redefined # mypy: ignore-errors """ @@ -76,7 +74,7 @@ class DispatchMeta(type): Dispatch abstract class for function registry """ - def __getattribute__(cls, func_name): # pylint: disable=redefined-outer-name + def __getattribute__(cls, func_name): if func_name in type.__getattribute__(cls, "registry").get(cls, {}): def dynamic_dispatch(*args: "Expression"): @@ -94,7 +92,7 @@ class Dispatch(metaclass=DispatchMeta): registry: ClassVar[Dict[str, Dict[Tuple[Tuple[int, Type]], Callable]]] = {} @classmethod - def register(cls, func): # pylint: disable=redefined-outer-name + def register(cls, func): func_name = func.__name__ params = inspect.signature(func).parameters spread_types = [[]] @@ -129,9 +127,7 @@ def register(cls, func): # pylint: disable=redefined-outer-name return func @classmethod - def dispatch( # pylint: disable=redefined-outer-name - cls, func_name, *args: "Expression" - ): + def dispatch(cls, func_name, *args: "Expression"): type_registry = cls.registry[cls].get(func_name) # type: ignore if not type_registry: raise ValueError( @@ -160,7 +156,7 @@ def dispatch( # pylint: disable=redefined-outer-name ) -class Function(Dispatch): # pylint: disable=too-few-public-methods +class Function(Dispatch): """ A DJ function. """ @@ -182,7 +178,7 @@ def compile_lambda(*args): """ -class TableFunction(Dispatch): # pylint: disable=too-few-public-methods +class TableFunction(Dispatch): """ A DJ table-valued function. """ @@ -274,9 +270,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `aggregate` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast # aggregate's lambda function can take three or four arguments, depending on whether # an optional finish function is provided @@ -1694,7 +1688,7 @@ def infer_type( return ct.DoubleType() -class E(Function): # pylint: disable=invalid-name +class E(Function): """ e() - Returns the mathematical constant e. """ @@ -1797,9 +1791,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `filter` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast expr, func = args if len(func.identifiers) != 1: @@ -1920,9 +1912,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `filter` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast expr, func = args if len(func.identifiers) > 2: @@ -2063,7 +2053,7 @@ def infer_type( _target_scale: ct.IntegerType, ) -> ct.DecimalType: target_scale = _target_scale.value - if isinstance(args.type, ct.DecimalType): # pylint: disable=R1705 + if isinstance(args.type, ct.DecimalType): precision = max(args.type.precision - args.type.scale + 1, -target_scale + 1) scale = min(args.type.scale, max(0, target_scale)) return ct.DecimalType(precision, scale) @@ -2104,9 +2094,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `filter` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast expr, func = args if len(func.identifiers) != 1: @@ -2167,11 +2155,10 @@ def infer_type( schema: ct.StringType, arg3: Optional[ct.MapType] = None, ) -> ct.ColumnType: - # TODO: Handle options? # pylint: disable=fixme - # pylint: disable=import-outside-toplevel + # TODO: Handle options? from datajunction_server.sql.parsing.backends.antlr4 import ( - parse_rule, # pragma: no cover - ) + parse_rule, + ) # pragma: no cover return ct.StructType( *parse_rule(schema.value, "complexColTypeList") @@ -2190,7 +2177,6 @@ def infer_type( schema: ct.StringType, options: Optional[Function] = None, ) -> ct.StructType: - # pylint: disable=import-outside-toplevel from datajunction_server.sql.parsing.backends.antlr4 import parse_rule schema_type = re.sub(r"^'(.*)'$", r"\1", schema.value) @@ -2335,9 +2321,7 @@ class HistogramNumeric(Function): @HistogramNumeric.register # type: ignore def infer_type(arg1: ct.ColumnType, arg2: ct.IntegerType) -> ct.ColumnType: # assuming that there's a StructType for the bin and frequency - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast return ct.ListType( element_type=ct.StructType( @@ -2955,7 +2939,7 @@ class MakeTimestamp(Function): @MakeTimestamp.register # type: ignore -def infer_type( # pylint: disable=too-many-arguments +def infer_type( year: ct.IntegerType, month: ct.IntegerType, day: ct.IntegerType, @@ -2974,7 +2958,7 @@ class MakeTimestampLtz(Function): @MakeTimestampLtz.register # type: ignore -def infer_type( # pylint: disable=too-many-arguments +def infer_type( year: ct.IntegerType, month: ct.IntegerType, day: ct.IntegerType, @@ -2994,7 +2978,7 @@ class MakeTimestampNtz(Function): @MakeTimestampNtz.register # type: ignore -def infer_type( # pylint: disable=too-many-arguments +def infer_type( year: ct.IntegerType, month: ct.IntegerType, day: ct.IntegerType, @@ -3082,9 +3066,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `map_filter` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast expr, func = args if len(func.identifiers) != 2: @@ -3170,9 +3152,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `map_zip_with` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast map1, map2, func = args available_identifiers = { @@ -3303,7 +3283,7 @@ def infer_type(arg: ct.NumberType) -> ct.ColumnType: return ct.DoubleType() -# TODO: fix parsing of: # pylint: disable=fixme +# TODO: fix parsing of: # SELECT median(col) FROM VALUES (INTERVAL '0' MONTH), # (INTERVAL '10' MONTH) AS tab(col) # in order to test this @@ -4165,7 +4145,7 @@ def infer_type( return ct.DoubleType() -class ToDate(Function): # pragma: no cover # pylint: disable=abstract-method +class ToDate(Function): # pragma: no cover """ Converts a date string to a date value. """ @@ -4179,7 +4159,7 @@ def infer_type( return ct.DateType() -class ToTimestamp(Function): # pragma: no cover # pylint: disable=abstract-method +class ToTimestamp(Function): # pragma: no cover """ Parses the timestamp_str expression with the fmt expression to a timestamp. """ @@ -4205,9 +4185,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `transform` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast expr, func = args available_identifiers = { @@ -4248,9 +4226,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `transform_keys` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast expr, func = args available_identifiers = { @@ -4291,9 +4267,7 @@ def compile_lambda(*args): Compiles the lambda function used by the `transform_values` Spark function so that the lambda's expression can be evaluated to determine the result's type. """ - from datajunction_server.sql.parsing import ( # pylint: disable=import-outside-toplevel - ast, - ) + from datajunction_server.sql.parsing import ast expr, func = args available_identifiers = { diff --git a/datajunction-server/datajunction_server/sql/parsing/ast.py b/datajunction-server/datajunction_server/sql/parsing/ast.py index aa61bd16c..3f2183942 100644 --- a/datajunction-server/datajunction_server/sql/parsing/ast.py +++ b/datajunction-server/datajunction_server/sql/parsing/ast.py @@ -1,5 +1,3 @@ -# pylint: disable=R0401,C0302 -# pylint: skip-file # mypy: ignore-errors import collections import decimal @@ -95,7 +93,7 @@ class CompileContext: # typevar used for node methods that return self # so the typesystem can correlate the self type with the return type -TNode = TypeVar("TNode", bound="Node") # pylint: disable=C0103 +TNode = TypeVar("TNode", bound="Node") class Node(ABC): @@ -231,7 +229,6 @@ def flatten(self) -> Iterator["Node"]: """ return self.filter(lambda _: True) - # pylint: disable=R0913 def fields( self, flat: bool = True, @@ -296,7 +293,7 @@ def make_child_generator(): else (child[1] is not None), child_generator, ), - ) # pylint: disable=C0301 + ) return child_generator @@ -314,7 +311,7 @@ def children(self) -> Iterator["Node"]: named=False, ) - def replace( # pylint: disable=invalid-name + def replace( self, from_: "Node", to: "Node", @@ -383,7 +380,7 @@ def compare( """ Compare two ASTs for deep equality """ - if type(self) != type(other): # pylint: disable=unidiomatic-typecheck + if type(self) != type(other): return False if id(self) == id(other): return True @@ -433,10 +430,8 @@ def __eq__(self, other) -> bool: Compares all others for type equality only. No recursing. Note: Does not check (sub)AST. See `Node.compare` for comparing (sub)ASTs. """ - return type(self) == type(other) and all( # pylint: disable=C0123 - s == o - if type(s) in PRIMITIVES # pylint: disable=C0123 - else type(s) == type(o) # pylint: disable=C0123 + return type(self) == type(other) and all( + s == o if type(s) in PRIMITIVES else type(s) == type(o) for s, o in zip( (self.fields(False, False, False, True)), (other.fields(False, False, False, True)), @@ -540,7 +535,7 @@ def alias_or_name(self) -> "Name": raise DJParseException("Node has no alias or name.") -AliasedType = TypeVar("AliasedType", bound=Node) # pylint: disable=C0103 +AliasedType = TypeVar("AliasedType", bound=Node) @dataclass(eq=False) @@ -570,7 +565,7 @@ def columns(self): return [self] -TExpression = TypeVar("TExpression", bound="Expression") # pylint: disable=C0103 +TExpression = TypeVar("TExpression", bound="Expression") @dataclass(eq=False) @@ -651,10 +646,10 @@ def identifier(self, quotes: bool = True) -> str: """ quote_style = "" if not quotes else self.quote_style namespace = str(self.namespace) + "." if self.namespace else "" - return f"{namespace}{quote_style}{self.name}{quote_style}" # pylint: disable=C0301 + return f"{namespace}{quote_style}{self.name}{quote_style}" -TNamed = TypeVar("TNamed", bound="Named") # pylint: disable=C0103 +TNamed = TypeVar("TNamed", bound="Named") @dataclass(eq=False) # type: ignore @@ -1411,7 +1406,6 @@ class Operation(Expression): """ -# pylint: disable=C0103 class UnaryOpKind(DJEnum): """ The accepted unary operations @@ -1460,7 +1454,6 @@ def raise_unop_exception(): raise DJParseException(f"Unary operation {self.op} not supported!") -# pylint: disable=C0103 class BinaryOpKind(DJEnum): """ The DJ AST accepted binary operations @@ -1501,7 +1494,7 @@ class BinaryOp(Operation): use_alias_as_name: Optional[bool] = False @classmethod - def And( # pylint: disable=invalid-name,keyword-arg-before-vararg + def And( cls, left: Expression, right: Optional[Expression] = None, @@ -1522,7 +1515,7 @@ def And( # pylint: disable=invalid-name,keyword-arg-before-vararg ) @classmethod - def Eq( # pylint: disable=invalid-name + def Eq( cls, left: Expression, right: Optional[Expression], @@ -1591,7 +1584,7 @@ def resolve_numeric_types_binary_operations( return left return left - BINOP_TYPE_COMBO_LOOKUP: Dict[ # pylint: disable=C0103 + BINOP_TYPE_COMBO_LOOKUP: Dict[ BinaryOpKind, Callable[[ColumnType, ColumnType], ColumnType], ] = { @@ -1694,7 +1687,6 @@ def __str__(self) -> str: return f"OVER ({consolidated_by}{window_frame})" -# pylint: disable=C0103 class SetQuantifier(DJEnum): """ The accepted set quantifiers @@ -1781,7 +1773,7 @@ async def compile(self, ctx: CompileContext): await arg.compile(ctx) arg._is_compiled = True - # FIXME: We currently catch this exception because we are unable # pylint: disable=fixme + # FIXME: We currently catch this exception because we are unable # to infer types for nested lambda functions. For the time being, an easy workaround is to # add a CAST(...) wrapper around the nested lambda function so that the type hard-coded by # the argument to CAST diff --git a/datajunction-server/datajunction_server/sql/parsing/backends/antlr4.py b/datajunction-server/datajunction_server/sql/parsing/backends/antlr4.py index 38d4d4939..7616c600a 100644 --- a/datajunction-server/datajunction_server/sql/parsing/backends/antlr4.py +++ b/datajunction-server/datajunction_server/sql/parsing/backends/antlr4.py @@ -1,4 +1,3 @@ -# pylint: skip-file # mypy: ignore-errors import copy import inspect @@ -34,14 +33,14 @@ class RemoveIdentifierBackticks(antlr4.ParseTreeListener): @staticmethod - def exitQuotedIdentifier(ctx): # pylint: disable=invalid-name,unused-argument + def exitQuotedIdentifier(ctx): def identity(token): return token return identity @staticmethod - def enterNonReserved(ctx): # pylint: disable=invalid-name,unused-argument + def enterNonReserved(ctx): def add_backtick(token): return "`{0}`".format(token) @@ -57,7 +56,7 @@ def syntaxError( column, msg, e, - ): # pylint: disable=invalid-name,no-self-use,too-many-arguments + ): raise SqlSyntaxError(f"Parse error {line}:{column}:", msg) @@ -70,12 +69,12 @@ class UpperCaseCharStream: def __init__(self, wrapped): self.wrapped = wrapped - def getText(self, interval, *args): # pylint: disable=invalid-name + def getText(self, interval, *args): if args or (self.size() > 0 and (interval.b - interval.a >= 0)): return self.wrapped.getText(interval, *args) return "" - def LA(self, i: int): # pylint: disable=invalid-name + def LA(self, i: int): token = self.wrapped.LA(i) if token in (0, -1): return token diff --git a/datajunction-server/datajunction_server/sql/parsing/types.py b/datajunction-server/datajunction_server/sql/parsing/types.py index 0d1dbd8c5..9aa3e1413 100644 --- a/datajunction-server/datajunction_server/sql/parsing/types.py +++ b/datajunction-server/datajunction_server/sql/parsing/types.py @@ -29,14 +29,14 @@ VARCHAR_PARSER = re.compile(r"(?i)varchar(\((?P\d+)\))?") -class Singleton: # pylint: disable=too-few-public-methods +class Singleton: """ Singleton for types """ _instance = None - def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument + def __new__(cls, *args, **kwargs): if not isinstance(cls._instance, cls): cls._instance = super(Singleton, cls).__new__(cls) return cls._instance @@ -49,12 +49,12 @@ class ColumnType(BaseModel): _initialized = False - class Config: # pylint: disable=missing-class-docstring, too-few-public-methods + class Config: extra = Extra.allow arbitrary_types_allowed = True underscore_attrs_are_private = False - def __init__( # pylint: disable=keyword-arg-before-vararg + def __init__( self, type_string: str, repr_string: str = None, @@ -85,16 +85,14 @@ def __get_validators__(cls) -> Generator[AnyCallable, None, None]: yield cls.validate @classmethod - def validate( # pylint: disable=too-many-return-statements + def validate( cls, v: Any, ) -> "ColumnType": """ Parses the column type """ - from datajunction_server.sql.parsing.backends.antlr4 import ( # pylint: disable=import-outside-toplevel - parse_rule, - ) + from datajunction_server.sql.parsing.backends.antlr4 import parse_rule return cast(ColumnType, parse_rule(str(v), "dataType")) @@ -144,15 +142,15 @@ def has_common_ancestor(type1, type2) -> bool: return has_common_ancestor(self.__class__, other.__class__) -class PrimitiveType(ColumnType): # pylint: disable=too-few-public-methods +class PrimitiveType(ColumnType): """Base class for all Column Primitive Types""" -class NumberType(PrimitiveType): # pylint: disable=too-few-public-methods +class NumberType(PrimitiveType): """Base class for all Column Number Types""" -class NullType(PrimitiveType, Singleton): # pylint: disable=too-few-public-methods +class NullType(PrimitiveType, Singleton): """A data type for NULL Example: @@ -264,9 +262,7 @@ def __new__( doc: Optional[str] = None, ): if isinstance(name, str): # pragma: no cover - from datajunction_server.sql.parsing.ast import ( # pylint: disable=import-outside-toplevel - Name, - ) + from datajunction_server.sql.parsing.ast import Name name = Name(name) @@ -283,9 +279,7 @@ def __init__( ): if not self._initialized: if isinstance(name, str): # pragma: no cover - from datajunction_server.sql.parsing.ast import ( # pylint: disable=import-outside-toplevel - Name, - ) + from datajunction_server.sql.parsing.ast import Name name = Name(name) doc_string = "" if doc is None else f", doc={repr(doc)}" @@ -569,7 +563,7 @@ def __init__(self): super().__init__("tinyint", "TinyIntType()") -class SmallIntType(IntegerBase): # pylint: disable=R0901 +class SmallIntType(IntegerBase): """A SmallInt data type can be represented using an instance of this class. SmallInts are 16-bit signed integers. @@ -617,7 +611,7 @@ def __init__(self): super().__init__("bigint", "BigIntType()") -class LongType(BigIntType): # pylint: disable=R0901 +class LongType(BigIntType): """A Long data type can be represented using an instance of this class. Longs are 64-bit signed integers. @@ -678,7 +672,6 @@ class DateTimeBase(PrimitiveType, Singleton): Base class for date and time types. """ - # pylint: disable=invalid-name class Unit(StrEnum): """ Units used for date and time functions and intervals @@ -696,8 +689,6 @@ class Unit(StrEnum): hour = "HOUR" millisecond = "MILLISECOND" - # pylint: enable=invalid-name - class DateType(DateTimeBase): """A Date data type can be represented using an instance of this class. Dates are @@ -796,11 +787,11 @@ def __init__( self._to = to_ @property - def from_(self) -> str: # pylint: disable=missing-function-docstring + def from_(self) -> str: return self._from # pragma: no cover @property - def to_( # pylint: disable=missing-function-docstring + def to_( self, ) -> Optional[str]: return self._to # pragma: no cover @@ -843,11 +834,11 @@ def __init__( self._to = to_ @property - def from_(self) -> str: # pylint: disable=missing-function-docstring + def from_(self) -> str: return self._from # pragma: no cover @property - def to_( # pylint: disable=missing-function-docstring + def to_( self, ) -> Optional[str]: return self._to # pragma: no cover diff --git a/datajunction-server/datajunction_server/superset.py b/datajunction-server/datajunction_server/superset.py index eb4979dd6..2ebf5124d 100644 --- a/datajunction-server/datajunction_server/superset.py +++ b/datajunction-server/datajunction_server/superset.py @@ -44,7 +44,7 @@ class MetricType(TypedDict, total=False): extra: Optional[str] -class DJEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method +class DJEngineSpec(BaseEngineSpec): """ Engine spec for the DataJunction metric repository @@ -69,7 +69,7 @@ class DJEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method } @classmethod - def select_star( # pylint: disable=unused-argument + def select_star( cls, *args: Any, **kwargs: Any, @@ -83,7 +83,7 @@ def select_star( # pylint: disable=unused-argument return f"SELECT '{message}' AS warning" @classmethod - def get_metrics( # pylint: disable=unused-argument + def get_metrics( cls, database: "Database", inspector: Inspector, @@ -125,7 +125,7 @@ def execute( return super().execute(cursor, query, **kwargs) @classmethod - def get_view_names( # pylint: disable=unused-argument + def get_view_names( cls, database: "Database", inspector: Inspector, diff --git a/datajunction-server/datajunction_server/transpilation.py b/datajunction-server/datajunction_server/transpilation.py index a492ffe45..b0b937449 100644 --- a/datajunction-server/datajunction_server/transpilation.py +++ b/datajunction-server/datajunction_server/transpilation.py @@ -64,7 +64,7 @@ def transpile_sql( """ Transpile a given SQL query using the specific library. """ - import sqlglot # pylint: disable=import-outside-toplevel,import-error + import sqlglot # Check to make sure that the output dialect is supported by sqlglot before transpiling if ( diff --git a/datajunction-server/datajunction_server/typing.py b/datajunction-server/datajunction_server/typing.py index 1b9cdf218..f3aae7119 100644 --- a/datajunction-server/datajunction_server/typing.py +++ b/datajunction-server/datajunction_server/typing.py @@ -2,8 +2,6 @@ Custom types for annotations. """ -# pylint: disable=missing-class-docstring - from __future__ import annotations import datetime @@ -16,7 +14,7 @@ from datajunction_server.enum import StrEnum -class SQLADialect(Protocol): # pylint: disable=too-few-public-methods +class SQLADialect(Protocol): """ A SQLAlchemy dialect. """ diff --git a/datajunction-server/datajunction_server/utils.py b/datajunction-server/datajunction_server/utils.py index 3e0bee5e7..8da8e42ef 100644 --- a/datajunction-server/datajunction_server/utils.py +++ b/datajunction-server/datajunction_server/utils.py @@ -9,7 +9,6 @@ from functools import lru_cache from http import HTTPStatus -# pylint: disable=line-too-long from typing import AsyncIterator, List, Optional from dotenv import load_dotenv @@ -163,7 +162,7 @@ async def get_session() -> AsyncIterator[AsyncSession]: session = session_manager.session() try: yield session - except Exception as exc: # pylint: disable=broad-exception-raised + except Exception as exc: await session.rollback() # pragma: no cover raise exc # pragma: no cover finally: @@ -186,7 +185,7 @@ async def refresh_if_needed(session: AsyncSession, obj, attributes: list[str]): await session.refresh(obj, attributes_to_refresh) -def get_query_service_client( # pylint: disable=unused-argument +def get_query_service_client( request: Request = None, ) -> Optional[QueryServiceClient]: """ diff --git a/datajunction-server/scripts/generate-openapi.py b/datajunction-server/scripts/generate-openapi.py index affbfaccd..71857035a 100755 --- a/datajunction-server/scripts/generate-openapi.py +++ b/datajunction-server/scripts/generate-openapi.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -# pylint: skip-file - import argparse import json diff --git a/datajunction-server/tests/api/access_test.py b/datajunction-server/tests/api/access_test.py index 4904ec94e..6d7749c05 100644 --- a/datajunction-server/tests/api/access_test.py +++ b/datajunction-server/tests/api/access_test.py @@ -10,7 +10,7 @@ from datajunction_server.models import access -class TestDataAccessControl: # pylint: disable=too-few-public-methods +class TestDataAccessControl: """ Test the data access control. """ diff --git a/datajunction-server/tests/api/client_test.py b/datajunction-server/tests/api/client_test.py index 214df7493..ec0c97747 100644 --- a/datajunction-server/tests/api/client_test.py +++ b/datajunction-server/tests/api/client_test.py @@ -33,7 +33,7 @@ def trim_trailing_whitespace(string: str) -> str: @pytest.mark.asyncio async def test_generated_python_client_code_new_source( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Test generating Python client code for creating a new source @@ -54,7 +54,7 @@ async def test_generated_python_client_code_new_source( @pytest.mark.asyncio async def test_generated_python_client_code_new_transform( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Test generating Python client code for creating a new transform @@ -82,7 +82,7 @@ async def test_generated_python_client_code_new_transform( @pytest.mark.asyncio async def test_generated_python_client_code_new_dimension( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Test generating Python client code for creating a new dimension @@ -109,7 +109,7 @@ async def test_generated_python_client_code_new_dimension( @pytest.mark.asyncio async def test_generated_python_client_code_new_metric( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Test generating Python client code for creating a new metric @@ -136,7 +136,7 @@ async def test_generated_python_client_code_new_metric( @pytest.mark.asyncio async def test_generated_python_client_code_new_cube( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Test generating Python client code for creating a new cube @@ -177,7 +177,7 @@ async def test_generated_python_client_code_new_cube( @pytest.mark.asyncio async def test_generated_python_client_code_link_dimension( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Test generating Python client code for creating a new dimension @@ -207,7 +207,7 @@ async def test_generated_python_client_code_link_dimension( @pytest.mark.asyncio async def test_include_client_setup( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Generate create new node python client code with client setup included. @@ -225,7 +225,7 @@ async def test_include_client_setup( @pytest.mark.asyncio async def test_export_namespace_as_notebook( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Verify exporting all nodes in a namespace as a notebook. @@ -304,7 +304,7 @@ async def test_export_namespace_as_notebook( @pytest.mark.asyncio async def test_export_cube_as_notebook( module__client_with_roads: AsyncClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Verify exporting all nodes relevant for a cube as a notebook. diff --git a/datajunction-server/tests/api/collections_test.py b/datajunction-server/tests/api/collections_test.py index 212141b35..82acfd066 100644 --- a/datajunction-server/tests/api/collections_test.py +++ b/datajunction-server/tests/api/collections_test.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines """ Tests for the collections API. """ diff --git a/datajunction-server/tests/api/cubes_test.py b/datajunction-server/tests/api/cubes_test.py index 9b3c897b1..428f776c0 100644 --- a/datajunction-server/tests/api/cubes_test.py +++ b/datajunction-server/tests/api/cubes_test.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines,redefined-outer-name,line-too-long """ Tests for the cubes API. """ @@ -879,7 +878,7 @@ async def test_cube_materialization_sql_and_measures( INNER JOIN default_DOT_hard_hat ON default_DOT_repair_order.hard_hat_id = default_DOT_hard_hat.hard_hat_id INNER JOIN default_DOT_dispatcher ON default_DOT_repair_order.dispatcher_id = default_DOT_dispatcher.dispatcher_id INNER JOIN default_DOT_municipality_dim ON default_DOT_repair_order.municipality_id = default_DOT_municipality_dim.municipality_id - """ # noqa: W191,E101,C0301 + """ assert str(parse(data["materializations"][0]["config"]["query"])) == str( parse(expected_materialization_query), ) @@ -891,7 +890,7 @@ async def test_cube_materialization_sql_and_measures( @pytest.mark.asyncio async def test_druid_cube_agg_materialization( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, module__query_service_client: Iterator[QueryServiceClient], ): """ @@ -1277,7 +1276,7 @@ async def test_materialized_cube_sql( @pytest.mark.asyncio async def test_remove_dimension_link_invalidate_cube( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, ): """ Verify that removing a dimension link can invalidate a cube. @@ -1339,7 +1338,7 @@ async def test_remove_dimension_link_invalidate_cube( @pytest.mark.asyncio async def test_changing_node_upstream_from_cube( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, ): """ Verify changing nodes upstream from a cube @@ -1505,7 +1504,7 @@ def assert_updated_repairs_cube(data): @pytest.mark.asyncio async def test_updating_cube( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, ): """ Verify updating a cube @@ -1609,7 +1608,7 @@ async def test_updating_cube( @pytest.mark.asyncio async def test_updating_cube_with_existing_materialization( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, module__query_service_client: QueryServiceClient, ): """ @@ -1827,7 +1826,7 @@ async def test_updating_cube_with_existing_materialization( @pytest.mark.asyncio async def test_get_materialized_cube_dimension_sql( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, ): """ Test building SQL to get unique dimension values for a materialized cube @@ -1982,7 +1981,7 @@ async def test_get_materialized_cube_dimension_sql( @pytest.mark.asyncio async def test_get_unmaterialized_cube_dimensions_values( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, ): """ Test building SQL + getting data for dimension values for an unmaterialized cube diff --git a/datajunction-server/tests/api/data_test.py b/datajunction-server/tests/api/data_test.py index d2543cc10..f83f80ef9 100644 --- a/datajunction-server/tests/api/data_test.py +++ b/datajunction-server/tests/api/data_test.py @@ -1,9 +1,7 @@ -# pylint: disable=too-many-lines """ Tests for the data API. """ -# pylint: disable=too-many-lines,C0302 from typing import Dict, List, Optional from unittest import mock @@ -644,7 +642,7 @@ async def test_get_data_for_query_id( assert "Query foo-bar-baz not found." in yet_another_response.text -class TestAvailabilityState: # pylint: disable=too-many-public-methods +class TestAvailabilityState: """ Test ``POST /data/{node_name}/availability/``. """ @@ -968,7 +966,7 @@ async def test_that_update_at_timestamp_is_being_updated( "default.large_revenue_payments_and_business_only", ) updated_at_1 = ( - large_revenue_payments_and_business_only.current.availability.updated_at # type: ignore # pylint: disable=line-too-long + large_revenue_payments_and_business_only.current.availability.updated_at # type: ignore ) response = await module__client_with_account_revenue.post( @@ -989,7 +987,7 @@ async def test_that_update_at_timestamp_is_being_updated( "default.large_revenue_payments_and_business_only", ) updated_at_2 = ( - large_revenue_payments_and_business_only.current.availability.updated_at # type: ignore # pylint: disable=line-too-long + large_revenue_payments_and_business_only.current.availability.updated_at # type: ignore ) assert updated_at_2 > updated_at_1 diff --git a/datajunction-server/tests/api/dimension_links_test.py b/datajunction-server/tests/api/dimension_links_test.py index 966e8dcb4..fc905203c 100644 --- a/datajunction-server/tests/api/dimension_links_test.py +++ b/datajunction-server/tests/api/dimension_links_test.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines """ Dimension linking related tests. @@ -31,7 +30,7 @@ async def dimensions_link_client(client: AsyncClient) -> AsyncClient: @pytest.mark.asyncio async def test_link_dimension_with_errors( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, ): """ Test linking dimensions with errors @@ -77,7 +76,7 @@ async def test_link_dimension_with_errors( @pytest.fixture def link_events_to_users_without_role( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, ): """ Link events with the users dimension without a role @@ -103,7 +102,7 @@ async def _link_events_to_users_without_role() -> Response: @pytest.fixture def link_events_to_users_with_role_direct( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, ): """ Link events with the users dimension with the role "user_direct", @@ -132,7 +131,7 @@ async def _link_events_to_users_with_role_direct() -> Response: @pytest.fixture def link_events_to_users_with_role_windowed( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, ): """ Link events with the users dimension with the role "user_windowed", @@ -159,7 +158,7 @@ async def _link_events_to_users_with_role_windowed() -> Response: @pytest.fixture def link_users_to_countries_with_role_registration( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, ): """ Link users to the countries dimension with role "registration_country". @@ -183,7 +182,7 @@ async def _link_users_to_countries_with_role_registration() -> Response: @pytest.fixture def reference_link_events_user_registration_country( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, ): """ Link users to the countries dimension with role "registration_country". @@ -205,8 +204,8 @@ async def _reference_link_events_user_registration_country() -> Response: @pytest.mark.asyncio async def test_link_complex_dimension_without_role( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name, - link_events_to_users_without_role, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, + link_events_to_users_without_role, ): """ Test linking complex dimension without role @@ -338,10 +337,10 @@ async def test_link_complex_dimension_without_role( @pytest.mark.asyncio async def test_link_complex_dimension_with_role( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name - link_events_to_users_with_role_direct, # pylint: disable=redefined-outer-name - link_events_to_users_with_role_windowed, # pylint: disable=redefined-outer-name - link_users_to_countries_with_role_registration, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, + link_events_to_users_with_role_direct, + link_events_to_users_with_role_windowed, + link_users_to_countries_with_role_registration, ): """ Testing linking complex dimension with roles. @@ -569,7 +568,6 @@ async def test_link_complex_dimension_with_role( # Get SQL for the downstream metric grouped by the user's registration country and # filtered by the user's residence country - # TODO # pylint: disable=fixme response = await dimensions_link_client.get( "/sql/default.elapsed_secs?", params={ @@ -633,9 +631,9 @@ async def test_link_complex_dimension_with_role( @pytest.mark.asyncio async def test_remove_dimension_link( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name - link_events_to_users_with_role_direct, # pylint: disable=redefined-outer-name - link_events_to_users_without_role, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, + link_events_to_users_with_role_direct, + link_events_to_users_without_role, ): """ Test removing complex dimension links @@ -703,10 +701,10 @@ async def test_remove_dimension_link( @pytest.mark.asyncio async def test_measures_sql_with_dimension_roles( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name - link_events_to_users_with_role_direct, # pylint: disable=redefined-outer-name - link_events_to_users_with_role_windowed, # pylint: disable=redefined-outer-name - link_users_to_countries_with_role_registration, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, + link_events_to_users_with_role_direct, + link_events_to_users_with_role_windowed, + link_users_to_countries_with_role_registration, ): """ Test measures SQL with dimension roles @@ -758,7 +756,7 @@ async def test_measures_sql_with_dimension_roles( INNER JOIN default_DOT_countries ON default_DOT_users.registration_country = default_DOT_countries.country_code""" assert str(parse(query)) == str(parse(expected)) - # TODO Implement caching for v2 measures SQL endpoint # pylint: disable=fixme + # TODO Implement caching for v2 measures SQL endpoint # query_request = ( # ( # await session.execute( @@ -788,8 +786,8 @@ async def test_measures_sql_with_dimension_roles( @pytest.mark.asyncio async def test_reference_dimension_links_errors( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name - reference_link_events_user_registration_country, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, + reference_link_events_user_registration_country, ): """ Test various reference dimension link errors @@ -843,9 +841,9 @@ async def test_reference_dimension_links_errors( @pytest.mark.asyncio async def test_measures_sql_with_reference_dimension_links( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name - reference_link_events_user_registration_country, # pylint: disable=redefined-outer-name - link_events_to_users_without_role, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, + reference_link_events_user_registration_country, + link_events_to_users_without_role, ): """ Test measures SQL generation with reference dimension links @@ -966,7 +964,7 @@ async def test_measures_sql_with_reference_dimension_links( @pytest.mark.asyncio async def test_dimension_link_cross_join( - dimensions_link_client: AsyncClient, # pylint: disable=redefined-outer-name + dimensions_link_client: AsyncClient, ): """ Testing linking complex dimension with CROSS JOIN as the join type. diff --git a/datajunction-server/tests/api/dimensions_access_test.py b/datajunction-server/tests/api/dimensions_access_test.py index 3af5d7377..e95fcebdf 100644 --- a/datajunction-server/tests/api/dimensions_access_test.py +++ b/datajunction-server/tests/api/dimensions_access_test.py @@ -15,12 +15,9 @@ async def test_list_nodes_with_dimension_access_limited( """ Test ``GET /dimensions/{name}/nodes/``. """ - # pylint: disable=import-outside-toplevel from datajunction_server.internal.access.authorization import validate_access from datajunction_server.models import access - # pylint: enable=import-outside-toplevel - def validate_access_override(): def _validate_access(access_control: access.AccessControl): for request in access_control.requests: diff --git a/datajunction-server/tests/api/djql_test.py b/datajunction-server/tests/api/djql_test.py index 995a8f78e..1eb90977e 100644 --- a/datajunction-server/tests/api/djql_test.py +++ b/datajunction-server/tests/api/djql_test.py @@ -7,8 +7,6 @@ from tests.sql.utils import assert_query_strings_equal, compare_query_strings -# pylint: disable=too-many-lines,C0301 - @pytest.mark.asyncio @pytest.mark.skip(reason="Will move djsql to new sql build later") diff --git a/datajunction-server/tests/api/graphql/common_dimensions_test.py b/datajunction-server/tests/api/graphql/common_dimensions_test.py index 089a9a475..4f166e321 100644 --- a/datajunction-server/tests/api/graphql/common_dimensions_test.py +++ b/datajunction-server/tests/api/graphql/common_dimensions_test.py @@ -44,7 +44,7 @@ def before_cursor_execute( @pytest.mark.asyncio async def test_get_common_dimensions( module__client_with_roads: AsyncClient, - capture_queries: AsyncGenerator[ # pylint: disable=redefined-outer-name + capture_queries: AsyncGenerator[ list[str], None, ], @@ -101,7 +101,7 @@ async def test_get_common_dimensions( @pytest.mark.asyncio async def test_get_common_dimensions_with_full_dim_node( module__client_with_roads: AsyncClient, - capture_queries: AsyncGenerator[ # pylint: disable=redefined-outer-name + capture_queries: AsyncGenerator[ list[str], None, ], diff --git a/datajunction-server/tests/api/graphql/find_nodes_test.py b/datajunction-server/tests/api/graphql/find_nodes_test.py index ecc1e3546..66fda2613 100644 --- a/datajunction-server/tests/api/graphql/find_nodes_test.py +++ b/datajunction-server/tests/api/graphql/find_nodes_test.py @@ -2,7 +2,6 @@ Tests for the engine API. """ -# pylint: disable=line-too-long,too-many-lines from unittest import mock import pytest diff --git a/datajunction-server/tests/api/graphql/tags_test.py b/datajunction-server/tests/api/graphql/tags_test.py index 3c59ab41b..faca710c0 100644 --- a/datajunction-server/tests/api/graphql/tags_test.py +++ b/datajunction-server/tests/api/graphql/tags_test.py @@ -69,7 +69,7 @@ async def client_with_tags( @pytest.mark.asyncio async def test_list_tags( - client_with_tags: AsyncClient, # pylint: disable=redefined-outer-name + client_with_tags: AsyncClient, ) -> None: """ Test listing tags @@ -126,7 +126,7 @@ async def test_list_tags( @pytest.mark.asyncio async def test_list_tag_types( - client_with_tags: AsyncClient, # pylint: disable=redefined-outer-name + client_with_tags: AsyncClient, ) -> None: """ Test listing tag types @@ -144,7 +144,7 @@ async def test_list_tag_types( @pytest.mark.asyncio async def test_find_tags_by_type( - client_with_tags: AsyncClient, # pylint: disable=redefined-outer-name + client_with_tags: AsyncClient, ) -> None: """ Test finding tags by tag type @@ -190,7 +190,7 @@ async def test_find_tags_by_type( @pytest.mark.asyncio async def test_find_tags_by_name( - client_with_tags: AsyncClient, # pylint: disable=redefined-outer-name + client_with_tags: AsyncClient, ) -> None: """ Test finding tags by tag type @@ -210,7 +210,7 @@ async def test_find_tags_by_name( @pytest.mark.asyncio async def test_tag_get_nodes( - client_with_tags: AsyncClient, # pylint: disable=redefined-outer-name + client_with_tags: AsyncClient, ) -> None: """ Test listing tags diff --git a/datajunction-server/tests/api/materializations_test.py b/datajunction-server/tests/api/materializations_test.py index 0ce3b968b..466991e6c 100644 --- a/datajunction-server/tests/api/materializations_test.py +++ b/datajunction-server/tests/api/materializations_test.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines """Tests for /materialization api""" import json @@ -44,7 +43,7 @@ def _load(filename: str): @pytest_asyncio.fixture async def client_with_repairs_cube( - module__client_with_roads: AsyncClient, # pylint: disable=redefined-outer-name + module__client_with_roads: AsyncClient, ): """ Adds a repairs cube to the test client @@ -83,7 +82,7 @@ async def _client_with_repairs_cube(cube_name: str = "default.repairs_cube"): @pytest.fixture -def set_temporal_column(): # pylint: disable=redefined-outer-name +def set_temporal_column(): """ Sets the given column as a temporal partition on the specified node. """ @@ -103,7 +102,7 @@ async def _set_temporal_column(client: AsyncClient, node_name: str, column: str) @pytest.fixture -def set_categorical_partition(): # pylint: disable=redefined-outer-name +def set_categorical_partition(): """ Sets the given column as a categorical partition on the specified node. """ @@ -301,10 +300,10 @@ async def test_crud_materialization(module__client_with_basic: AsyncClient): @pytest.mark.asyncio async def test_druid_measures_cube_full( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, module__query_service_client: QueryServiceClient, - load_expected_file, # pylint: disable=redefined-outer-name - set_temporal_column, # pylint: disable=redefined-outer-name + load_expected_file, + set_temporal_column, ): """ Verifying this materialization setup: @@ -421,11 +420,11 @@ async def test_druid_measures_cube_full( @pytest.mark.asyncio async def test_druid_measures_cube_incremental( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, module__query_service_client: QueryServiceClient, - load_expected_file, # pylint: disable=redefined-outer-name - set_temporal_column, # pylint: disable=redefined-outer-name - set_categorical_partition, # pylint: disable=redefined-outer-name, + load_expected_file, + set_temporal_column, + set_categorical_partition, ): """ Verifying this materialization setup: @@ -609,11 +608,11 @@ async def test_druid_measures_cube_incremental( @pytest.mark.asyncio @pytest.mark.skip(reason="The test is unstable depending on run order") async def test_druid_metrics_cube_incremental( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, module__query_service_client: QueryServiceClient, - load_expected_file, # pylint: disable=redefined-outer-name - set_temporal_column, # pylint: disable=redefined-outer-name, - set_categorical_partition, # pylint: disable=redefined-outer-name, + load_expected_file, + set_temporal_column, + set_categorical_partition, ): """ Verifying this materialization setup: @@ -748,9 +747,9 @@ def __repr__(self): @pytest.mark.asyncio async def test_druid_cube_incremental( - client_with_repairs_cube: AsyncClient, # pylint: disable=redefined-outer-name + client_with_repairs_cube: AsyncClient, module__query_service_client: QueryServiceClient, - set_temporal_column, # pylint: disable=redefined-outer-name + set_temporal_column, ): """ Verifying this materialization setup: @@ -1032,9 +1031,9 @@ async def test_druid_cube_incremental( @pytest.mark.asyncio async def test_spark_sql_full( - module__client_with_roads: AsyncClient, # pylint: disable=redefined-outer-name + module__client_with_roads: AsyncClient, module__query_service_client: QueryServiceClient, - load_expected_file, # pylint: disable=redefined-outer-name + load_expected_file, ): """ Verifying this materialization setup: @@ -1171,11 +1170,11 @@ async def test_spark_sql_full( @pytest.mark.asyncio async def test_spark_sql_incremental( - module__client_with_roads: AsyncClient, # pylint: disable=redefined-outer-name + module__client_with_roads: AsyncClient, module__query_service_client: QueryServiceClient, - set_temporal_column, # pylint: disable=redefined-outer-name - set_categorical_partition, # pylint: disable=redefined-outer-name - load_expected_file, # pylint: disable=redefined-outer-name + set_temporal_column, + set_categorical_partition, + load_expected_file, ): """ Verifying this materialization setup: @@ -1407,7 +1406,7 @@ async def test_spark_sql_incremental( @pytest.mark.asyncio async def test_spark_with_availablity( - module__client_with_roads: AsyncClient, # pylint: disable=redefined-outer-name + module__client_with_roads: AsyncClient, ): """ Verify that we build the Query correctly with or without the materialization availability. diff --git a/datajunction-server/tests/api/measures_test.py b/datajunction-server/tests/api/measures_test.py index 19a311758..dc9b029fa 100644 --- a/datajunction-server/tests/api/measures_test.py +++ b/datajunction-server/tests/api/measures_test.py @@ -100,7 +100,7 @@ async def test_list_all_measures( @pytest.mark.asyncio async def test_create_measure( module__client_with_roads: AsyncClient, - failed_measure: Dict, # pylint: disable=redefined-outer-name + failed_measure: Dict, ) -> None: """ Test ``POST /measures/``. diff --git a/datajunction-server/tests/api/metrics_test.py b/datajunction-server/tests/api/metrics_test.py index e248c2300..b6e1b46c0 100644 --- a/datajunction-server/tests/api/metrics_test.py +++ b/datajunction-server/tests/api/metrics_test.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines """ Tests for the metrics API. """ diff --git a/datajunction-server/tests/api/nodes_test.py b/datajunction-server/tests/api/nodes_test.py index bcac295f0..db5619a7e 100644 --- a/datajunction-server/tests/api/nodes_test.py +++ b/datajunction-server/tests/api/nodes_test.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines """ Tests for the nodes API. """ @@ -280,7 +279,7 @@ async def test_get_nodes_with_details(client_with_examples: AsyncClient): } -class TestNodeCRUD: # pylint: disable=too-many-public-methods +class TestNodeCRUD: """ Test node CRUD """ @@ -1291,7 +1290,7 @@ async def verify_complete_hard_delete( ( await session.execute( select(NodeRelationship).where( - NodeRelationship.child_id.in_( # type: ignore # pylint: disable=no-member + NodeRelationship.child_id.in_( # type: ignore [rev.id for rev in revisions], ), ), @@ -1310,7 +1309,7 @@ async def verify_complete_hard_delete( ( await session.execute( select(Node).where( - Node.name.in_(upstream_names), # type: ignore # pylint: disable=no-member + Node.name.in_(upstream_names), # type: ignore ), ) ) @@ -1373,7 +1372,7 @@ async def test_hard_deleting_node_with_versions( ( await session.execute( select(NodeRevision).where( - NodeRevision.name.like("default%"), # type: ignore # pylint: disable=no-member + NodeRevision.name.like("default%"), # type: ignore ), ) ) @@ -2259,8 +2258,8 @@ async def test_raise_on_source_node_with_no_catalog( @pytest.mark.asyncio async def test_create_invalid_transform_node( self, - catalog: Catalog, # pylint: disable=unused-argument - source_node: Node, # pylint: disable=unused-argument + catalog: Catalog, + source_node: Node, client: AsyncClient, create_invalid_transform_node_payload: Dict[str, Any], ) -> None: @@ -2326,8 +2325,8 @@ async def test_create_node_with_type_inference_failure( @pytest.mark.asyncio async def test_create_update_transform_node( self, - catalog: Catalog, # pylint: disable=unused-argument - source_node: Node, # pylint: disable=unused-argument + catalog: Catalog, + source_node: Node, client: AsyncClient, create_transform_node_payload: Dict[str, Any], ) -> None: @@ -2590,8 +2589,8 @@ async def test_update_metric_node(self, client_with_roads: AsyncClient): @pytest.mark.asyncio async def test_create_dimension_node_fails( self, - catalog: Catalog, # pylint: disable=unused-argument - source_node: Node, # pylint: disable=unused-argument + catalog: Catalog, + source_node: Node, client: AsyncClient, ): """ @@ -2632,8 +2631,8 @@ async def test_create_dimension_node_fails( @pytest.mark.asyncio async def test_create_update_dimension_node( self, - catalog: Catalog, # pylint: disable=unused-argument - source_node: Node, # pylint: disable=unused-argument + catalog: Catalog, + source_node: Node, client: AsyncClient, create_dimension_node_payload: Dict[str, Any], ) -> None: @@ -2804,8 +2803,8 @@ async def test_raise_on_multi_catalog_node(self, client_example_loader): @pytest.mark.asyncio async def test_updating_node_to_invalid_draft( self, - catalog: Catalog, # pylint: disable=unused-argument - source_node: Node, # pylint: disable=unused-argument + catalog: Catalog, + source_node: Node, client: AsyncClient, create_dimension_node_payload: Dict[str, Any], ) -> None: @@ -2871,7 +2870,7 @@ async def test_updating_node_to_invalid_draft( assert data["status"] == "invalid" @pytest.mark.asyncio - async def test_upsert_materialization_config( # pylint: disable=too-many-arguments + async def test_upsert_materialization_config( self, client_with_query_service_example_loader, ) -> None: @@ -3840,7 +3839,7 @@ async def test_set_columns_attributes_failed(self, client_with_basic: AsyncClien ] -class TestValidateNodes: # pylint: disable=too-many-public-methods +class TestValidateNodes: """ Test ``POST /nodes/validate/``. """ @@ -5012,7 +5011,7 @@ async def test_resolving_downstream_status( (metric3, NodeType.METRIC), ]: response = await client_with_service_setup.post( - f"/nodes/{node_type.value}/", # pylint: disable=no-member + f"/nodes/{node_type.value}/", json=node, ) assert response.status_code == 201 @@ -5768,11 +5767,11 @@ async def test_copy_node_failures(self, client_with_roads: AsyncClient): ] @pytest.mark.asyncio - async def test_copy_nodes( # pylint: disable=too-many-locals + async def test_copy_nodes( self, client_with_roads: AsyncClient, - repairs_cube_payload, # pylint: disable=redefined-outer-name - metric_with_required_dim_payload, # pylint: disable=redefined-outer-name + repairs_cube_payload, + metric_with_required_dim_payload, ): """ Test copying all nodes in the roads database diff --git a/datajunction-server/tests/api/sql_test.py b/datajunction-server/tests/api/sql_test.py index b1dfa9dd5..188b15a4d 100644 --- a/datajunction-server/tests/api/sql_test.py +++ b/datajunction-server/tests/api/sql_test.py @@ -1,7 +1,5 @@ """Tests for the /sql/ endpoint""" -# pylint: disable=line-too-long,too-many-lines -# pylint: disable=C0302 import duckdb import pytest from httpx import AsyncClient, Response @@ -276,11 +274,11 @@ async def get_query_requests(session: AsyncSession, query_type: QueryBuildType): @pytest.mark.asyncio -async def test_saving_node_sql_requests( # pylint: disable=too-many-statements +async def test_saving_node_sql_requests( session: AsyncSession, client_with_roads: AsyncClient, - transform_node_sql_request, # pylint: disable=redefined-outer-name - update_transform_node, # pylint: disable=redefined-outer-name + transform_node_sql_request, + update_transform_node, ) -> None: """ Test different scenarios involving saving and reusing cached query requests when @@ -419,8 +417,8 @@ async def test_saving_node_sql_requests( # pylint: disable=too-many-statements async def test_saving_measures_sql_requests( session: AsyncSession, client_with_roads: AsyncClient, - measures_sql_request, # pylint: disable=redefined-outer-name - update_transform_node, # pylint: disable=redefined-outer-name + measures_sql_request, + update_transform_node, ) -> None: """ Test saving query request while requesting measures SQL for a set of metrics + dimensions @@ -501,11 +499,11 @@ async def test_saving_measures_sql_requests( @pytest.mark.asyncio -async def test_saving_metrics_sql_requests( # pylint: disable=too-many-statements +async def test_saving_metrics_sql_requests( session: AsyncSession, client_with_roads: AsyncClient, - metrics_sql_request, # pylint: disable=redefined-outer-name - update_transform_node, # pylint: disable=redefined-outer-name + metrics_sql_request, + update_transform_node, ) -> None: """ Requesting metrics SQL for a set of metrics + dimensions + filters @@ -627,7 +625,7 @@ async def test_saving_metrics_sql_requests( # pylint: disable=too-many-statemen assert len(query_request) == 3 -async def verify_node_sql( # pylint: disable=too-many-arguments +async def verify_node_sql( custom_client: AsyncClient, node_name: str, dimensions, @@ -656,7 +654,7 @@ async def verify_node_sql( # pylint: disable=too-many-arguments @pytest.mark.asyncio -async def test_transform_sql_filter_joinable_dimension( # pylint: disable=too-many-arguments +async def test_transform_sql_filter_joinable_dimension( module__client_with_examples: AsyncClient, ): """ @@ -885,7 +883,7 @@ async def test_transform_sql_filter_joinable_dimension( # pylint: disable=too-m @pytest.mark.asyncio -async def test_transform_sql_filter_dimension_pk_col( # pylint: disable=too-many-arguments +async def test_transform_sql_filter_dimension_pk_col( module__client_with_examples: AsyncClient, ): """ @@ -1067,7 +1065,7 @@ async def test_transform_sql_filter_dimension_pk_col( # pylint: disable=too-man @pytest.mark.asyncio -async def test_transform_sql_filter_direct_node( # pylint: disable=too-many-arguments +async def test_transform_sql_filter_direct_node( module__client_with_examples: AsyncClient, ): """ @@ -1631,7 +1629,7 @@ async def test_metric_with_node_level_and_nth_order_filters( Verify metric SQL generation with filters on dimensions at the metric's parent node level and filters on nth-order dimensions. """ - await verify_node_sql( # pylint: disable=expression-not-assigned + await verify_node_sql( custom_client=module__client_with_examples, node_name="default.num_repair_orders", dimensions=["default.hard_hat.state"], @@ -2408,191 +2406,6 @@ async def test_metric_with_joinable_dimension_multiple_hops( ) -# @pytest.mark.parametrize( -# "node_name, dimensions, filters, orderby, sql", -# [ -# # querying on source node with filter on joinable dimension -# ( -# "foo.bar.repair_orders", -# [], -# ["foo.bar.hard_hat.state='CA'"], -# [], -# """ -# SELECT foo_DOT_bar_DOT_repair_orders.repair_order_id foo_DOT_bar_DOT_repair_orders_DOT_repair_order_id, -# foo_DOT_bar_DOT_repair_orders.municipality_id foo_DOT_bar_DOT_repair_orders_DOT_municipality_id, -# foo_DOT_bar_DOT_repair_orders.hard_hat_id foo_DOT_bar_DOT_repair_orders_DOT_hard_hat_id, -# foo_DOT_bar_DOT_repair_orders.order_date foo_DOT_bar_DOT_repair_orders_DOT_order_date, -# foo_DOT_bar_DOT_repair_orders.required_date foo_DOT_bar_DOT_repair_orders_DOT_required_date, -# foo_DOT_bar_DOT_repair_orders.dispatched_date foo_DOT_bar_DOT_repair_orders_DOT_dispatched_date, -# foo_DOT_bar_DOT_repair_orders.dispatcher_id foo_DOT_bar_DOT_repair_orders_DOT_dispatcher_id, -# foo_DOT_bar_DOT_hard_hat.state foo_DOT_bar_DOT_hard_hat_DOT_state -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders LEFT JOIN (SELECT foo_DOT_bar_DOT_repair_orders.repair_order_id, -# foo_DOT_bar_DOT_repair_orders.municipality_id, -# foo_DOT_bar_DOT_repair_orders.hard_hat_id, -# foo_DOT_bar_DOT_repair_orders.dispatcher_id -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders) -# AS foo_DOT_bar_DOT_repair_order ON foo_DOT_bar_DOT_repair_orders.repair_order_id = foo_DOT_bar_DOT_repair_order.repair_order_id -# LEFT JOIN (SELECT foo_DOT_bar_DOT_hard_hats.hard_hat_id, -# foo_DOT_bar_DOT_hard_hats.state -# FROM roads.hard_hats AS foo_DOT_bar_DOT_hard_hats) -# AS foo_DOT_bar_DOT_hard_hat ON foo_DOT_bar_DOT_repair_order.hard_hat_id = foo_DOT_bar_DOT_hard_hat.hard_hat_id -# WHERE foo_DOT_bar_DOT_hard_hat.state = 'CA' -# """, -# ), -# # querying source node with filters directly on the node -# ( -# "foo.bar.repair_orders", -# [], -# ["foo.bar.repair_orders.order_date='2009-08-14'"], -# [], -# """ -# SELECT -# foo_DOT_bar_DOT_repair_orders.repair_order_id foo_DOT_bar_DOT_repair_orders_DOT_repair_order_id, -# foo_DOT_bar_DOT_repair_orders.municipality_id foo_DOT_bar_DOT_repair_orders_DOT_municipality_id, -# foo_DOT_bar_DOT_repair_orders.hard_hat_id foo_DOT_bar_DOT_repair_orders_DOT_hard_hat_id, -# foo_DOT_bar_DOT_repair_orders.order_date foo_DOT_bar_DOT_repair_orders_DOT_order_date, -# foo_DOT_bar_DOT_repair_orders.required_date foo_DOT_bar_DOT_repair_orders_DOT_required_date, -# foo_DOT_bar_DOT_repair_orders.dispatched_date foo_DOT_bar_DOT_repair_orders_DOT_dispatched_date, -# foo_DOT_bar_DOT_repair_orders.dispatcher_id foo_DOT_bar_DOT_repair_orders_DOT_dispatcher_id -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders -# WHERE -# foo_DOT_bar_DOT_repair_orders.order_date = '2009-08-14' -# """, -# ), -# ( -# "foo.bar.num_repair_orders", -# [], -# [], -# [], -# """ -# SELECT -# count(foo_DOT_bar_DOT_repair_orders.repair_order_id) AS foo_DOT_bar_DOT_num_repair_orders -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders -# """, -# ), -# ( -# "foo.bar.num_repair_orders", -# ["foo.bar.hard_hat.state"], -# ["foo.bar.repair_orders.dispatcher_id=1", "foo.bar.hard_hat.state='AZ'"], -# [], -# """ -# SELECT count(foo_DOT_bar_DOT_repair_orders.repair_order_id) AS foo_DOT_bar_DOT_num_repair_orders, -# foo_DOT_bar_DOT_hard_hat.state foo_DOT_bar_DOT_hard_hat_DOT_state -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders LEFT JOIN (SELECT foo_DOT_bar_DOT_repair_orders.repair_order_id, -# foo_DOT_bar_DOT_repair_orders.municipality_id, -# foo_DOT_bar_DOT_repair_orders.hard_hat_id, -# foo_DOT_bar_DOT_repair_orders.dispatcher_id -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders) -# AS foo_DOT_bar_DOT_repair_order ON foo_DOT_bar_DOT_repair_orders.repair_order_id = foo_DOT_bar_DOT_repair_order.repair_order_id -# LEFT JOIN (SELECT foo_DOT_bar_DOT_hard_hats.hard_hat_id, -# foo_DOT_bar_DOT_hard_hats.state -# FROM roads.hard_hats AS foo_DOT_bar_DOT_hard_hats) -# AS foo_DOT_bar_DOT_hard_hat ON foo_DOT_bar_DOT_repair_order.hard_hat_id = foo_DOT_bar_DOT_hard_hat.hard_hat_id -# WHERE foo_DOT_bar_DOT_hard_hat.state = 'AZ' AND foo_DOT_bar_DOT_repair_orders.dispatcher_id = 1 -# GROUP BY foo_DOT_bar_DOT_hard_hat.state -# """, -# ), -# ( -# "foo.bar.num_repair_orders", -# [ -# "foo.bar.hard_hat.city", -# "foo.bar.hard_hat.last_name", -# "foo.bar.dispatcher.company_name", -# "foo.bar.municipality_dim.local_region", -# ], -# [ -# "foo.bar.repair_orders.dispatcher_id=1", -# "foo.bar.hard_hat.state != 'AZ'", -# "foo.bar.dispatcher.phone = '4082021022'", -# "foo.bar.repair_orders.order_date >= '2020-01-01'", -# ], -# ["foo.bar.hard_hat.last_name"], -# """ -# SELECT count(foo_DOT_bar_DOT_repair_orders.repair_order_id) AS foo_DOT_bar_DOT_num_repair_orders, -# foo_DOT_bar_DOT_hard_hat.city foo_DOT_bar_DOT_hard_hat_DOT_city, -# foo_DOT_bar_DOT_hard_hat.last_name foo_DOT_bar_DOT_hard_hat_DOT_last_name, -# foo_DOT_bar_DOT_dispatcher.company_name foo_DOT_bar_DOT_dispatcher_DOT_company_name, -# foo_DOT_bar_DOT_municipality_dim.local_region foo_DOT_bar_DOT_municipality_dim_DOT_local_region -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders LEFT JOIN (SELECT foo_DOT_bar_DOT_repair_orders.repair_order_id, -# foo_DOT_bar_DOT_repair_orders.municipality_id, -# foo_DOT_bar_DOT_repair_orders.hard_hat_id, -# foo_DOT_bar_DOT_repair_orders.dispatcher_id -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders) -# AS foo_DOT_bar_DOT_repair_order ON foo_DOT_bar_DOT_repair_orders.repair_order_id = foo_DOT_bar_DOT_repair_order.repair_order_id -# LEFT JOIN (SELECT foo_DOT_bar_DOT_dispatchers.dispatcher_id, -# foo_DOT_bar_DOT_dispatchers.company_name, -# foo_DOT_bar_DOT_dispatchers.phone -# FROM roads.dispatchers AS foo_DOT_bar_DOT_dispatchers) -# AS foo_DOT_bar_DOT_dispatcher ON foo_DOT_bar_DOT_repair_order.dispatcher_id = foo_DOT_bar_DOT_dispatcher.dispatcher_id -# LEFT JOIN (SELECT foo_DOT_bar_DOT_hard_hats.hard_hat_id, -# foo_DOT_bar_DOT_hard_hats.last_name, -# foo_DOT_bar_DOT_hard_hats.city, -# foo_DOT_bar_DOT_hard_hats.state -# FROM roads.hard_hats AS foo_DOT_bar_DOT_hard_hats) -# AS foo_DOT_bar_DOT_hard_hat ON foo_DOT_bar_DOT_repair_order.hard_hat_id = foo_DOT_bar_DOT_hard_hat.hard_hat_id -# LEFT JOIN (SELECT m.municipality_id AS municipality_id, -# m.local_region -# FROM roads.municipality AS m LEFT JOIN roads.municipality_municipality_type AS mmt ON m.municipality_id = mmt.municipality_id -# LEFT JOIN roads.municipality_type AS mt ON mmt.municipality_type_id = mt.municipality_type_desc) -# AS foo_DOT_bar_DOT_municipality_dim ON foo_DOT_bar_DOT_repair_order.municipality_id = foo_DOT_bar_DOT_municipality_dim.municipality_id -# WHERE foo_DOT_bar_DOT_repair_orders.order_date >= '2020-01-01' AND foo_DOT_bar_DOT_dispatcher.phone = '4082021022' AND foo_DOT_bar_DOT_hard_hat.state != 'AZ' AND foo_DOT_bar_DOT_repair_orders.dispatcher_id = 1 -# GROUP BY foo_DOT_bar_DOT_hard_hat.city, foo_DOT_bar_DOT_hard_hat.last_name, foo_DOT_bar_DOT_dispatcher.company_name, foo_DOT_bar_DOT_municipality_dim.local_region -# ORDER BY foo_DOT_bar_DOT_hard_hat.last_name -# """, -# ), -# ( -# "foo.bar.avg_repair_price", -# ["foo.bar.hard_hat.city"], -# [], -# [], -# """ -# SELECT -# avg(foo_DOT_bar_DOT_repair_order_details.price) foo_DOT_bar_DOT_avg_repair_price, -# foo_DOT_bar_DOT_hard_hat.city foo_DOT_bar_DOT_hard_hat_DOT_city -# FROM roads.repair_order_details AS foo_DOT_bar_DOT_repair_order_details -# LEFT JOIN ( -# SELECT -# foo_DOT_bar_DOT_repair_orders.repair_order_id, -# foo_DOT_bar_DOT_repair_orders.municipality_id, -# foo_DOT_bar_DOT_repair_orders.hard_hat_id, -# foo_DOT_bar_DOT_repair_orders.dispatcher_id -# FROM roads.repair_orders AS foo_DOT_bar_DOT_repair_orders -# ) AS foo_DOT_bar_DOT_repair_order -# ON foo_DOT_bar_DOT_repair_order_details.repair_order_id -# = foo_DOT_bar_DOT_repair_order.repair_order_id -# LEFT JOIN ( -# SELECT -# foo_DOT_bar_DOT_hard_hats.hard_hat_id, -# foo_DOT_bar_DOT_hard_hats.city -# FROM roads.hard_hats AS foo_DOT_bar_DOT_hard_hats -# ) AS foo_DOT_bar_DOT_hard_hat -# ON foo_DOT_bar_DOT_repair_order.hard_hat_id = foo_DOT_bar_DOT_hard_hat.hard_hat_id -# GROUP BY foo_DOT_bar_DOT_hard_hat.city -# """, -# ), -# ], -# ) -# @pytest.mark.asyncio -# async def test_sql_with_filters_on_namespaced_nodes( # pylint: disable=R0913 -# node_name, -# dimensions, -# filters, -# orderby, -# sql, -# module__client_with_examples: AsyncClient, -# ): -# """ -# Test ``GET /sql/{node_name}/`` with various filters and dimensions using a -# version of the DJ roads database with namespaces. -# """ -# response = await module__client_with_examples.get( -# f"/sql/{node_name}/", -# params={"dimensions": dimensions, "filters": filters, "orderby": orderby}, -# ) -# data = response.json() -# assert str(parse(str(data["sql"]))) == str(parse(str(sql))) - - @pytest.mark.asyncio async def test_union_all( module__client_with_examples: AsyncClient, @@ -3309,7 +3122,7 @@ async def test_get_sql_including_dimension_ids( @pytest.mark.asyncio async def test_get_sql_including_dimensions_with_disambiguated_columns( module__client_with_examples: AsyncClient, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ): """ Test getting SQL that includes dimensions with SQL that has to disambiguate projection columns with prefixes @@ -3799,7 +3612,7 @@ async def test_filter_pushdowns( @pytest.mark.asyncio async def test_sql_use_materialized_table( - measures_sql_request, # pylint: disable=redefined-outer-name + measures_sql_request, module__client_with_examples: AsyncClient, ): """ diff --git a/datajunction-server/tests/api/sql_v2_test.py b/datajunction-server/tests/api/sql_v2_test.py index 2b1c1b2cf..e743f2bb6 100644 --- a/datajunction-server/tests/api/sql_v2_test.py +++ b/datajunction-server/tests/api/sql_v2_test.py @@ -1,6 +1,5 @@ """Tests for all /sql endpoints that use node SQL build v2""" -# pylint: disable=line-too-long,too-many-lines from unittest import mock import duckdb @@ -501,7 +500,7 @@ async def fix_dimension_links(module__client_with_roads: AsyncClient): ], ) @pytest.mark.asyncio -async def test_measures_sql_with_filters__v2( # pylint: disable=too-many-arguments +async def test_measures_sql_with_filters__v2( metrics, dimensions, filters, @@ -510,7 +509,7 @@ async def test_measures_sql_with_filters__v2( # pylint: disable=too-many-argume columns, rows, module__client_with_roads: AsyncClient, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ): """ Test ``GET /sql/measures`` with various metrics, filters, and dimensions. @@ -988,7 +987,7 @@ async def test_measures_sql_with_filters__v2( # pylint: disable=too-many-argume ], ) @pytest.mark.asyncio -async def test_measures_sql_preaggregate( # pylint: disable=too-many-arguments +async def test_measures_sql_preaggregate( metrics, dimensions, filters, @@ -997,7 +996,7 @@ async def test_measures_sql_preaggregate( # pylint: disable=too-many-arguments columns, rows, module__client_with_roads: AsyncClient, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ): """ Test ``GET /sql/measures`` with various metrics, filters, and dimensions. @@ -1026,7 +1025,7 @@ async def test_measures_sql_preaggregate( # pylint: disable=too-many-arguments @pytest.mark.asyncio async def test_measures_sql_include_all_columns( module__client_with_roads: AsyncClient, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ): """ Test ``GET /sql/measures/v2`` with include_all_columns set to true. @@ -1178,9 +1177,9 @@ async def test_measures_sql_errors( @pytest.mark.asyncio -async def test_measures_sql_preagg_incompatible( # pylint: disable=too-many-arguments +async def test_measures_sql_preagg_incompatible( module__client_with_roads: AsyncClient, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ): """ Test ``GET /sql/measures`` with incompatible metrics vs compatible metrics. @@ -1345,7 +1344,7 @@ async def test_measures_sql_preagg_incompatible( # pylint: disable=too-many-arg @pytest.mark.asyncio async def test_metrics_sql_different_parents( module__client_with_roads: AsyncClient, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ): """ Test ``GET /sql`` for metrics from different parents. @@ -1460,7 +1459,7 @@ async def test_metrics_sql_different_parents( @pytest.mark.asyncio async def test_measures_sql_local_dimensions( module__client_with_roads: AsyncClient, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ): """ Test measures SQL for metrics that reference local dimensions diff --git a/datajunction-server/tests/conftest.py b/datajunction-server/tests/conftest.py index c1792effc..0bf3e240a 100644 --- a/datajunction-server/tests/conftest.py +++ b/datajunction-server/tests/conftest.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines """ Fixtures for testing. """ @@ -58,8 +57,6 @@ from .examples import COLUMN_MAPPINGS, EXAMPLES, QUERY_DATA_MAPPINGS, SERVICE_SETUP -# pylint: disable=redefined-outer-name, invalid-name, W0611 - EXAMPLE_TOKEN = ( "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4R0NNIn0..SxGbG0NRepMY4z9-2-ZZdg.ug" @@ -133,16 +130,16 @@ def settings_no_qs( @pytest.fixture(scope="session") -def duckdb_conn() -> duckdb.DuckDBPyConnection: # pylint: disable=c-extension-no-member +def duckdb_conn() -> duckdb.DuckDBPyConnection: """ DuckDB connection fixture with mock roads data loaded """ - with open( # pylint: disable=unspecified-encoding + with open( os.path.join(os.path.dirname(__file__), "duckdb.sql"), ) as mock_data: - with duckdb.connect( # pylint: disable=c-extension-no-member + with duckdb.connect( ":memory:", - ) as conn: # pylint: disable=c-extension-no-member + ) as conn: conn.execute(mock_data.read()) yield conn @@ -216,7 +213,7 @@ def event_loop(): @pytest.fixture def query_service_client( mocker: MockerFixture, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ) -> Iterator[QueryServiceClient]: """ Custom settings for unit tests. @@ -227,10 +224,8 @@ def mock_get_columns_for_table( catalog: str, schema: str, table: str, - engine: Optional[Engine] = None, # pylint: disable=unused-argument - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + engine: Optional[Engine] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> List[Column]: return COLUMN_MAPPINGS[f"{catalog}.{schema}.{table}"] @@ -242,9 +237,7 @@ def mock_get_columns_for_table( def mock_submit_query( query_create: QueryCreate, - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> QueryWithResults: result = duckdb_conn.sql(query_create.submitted_query) columns = [ @@ -274,9 +267,7 @@ def mock_submit_query( def mock_create_view( view_name: str, query_create: QueryCreate, - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> str: duckdb_conn.sql(query_create.submitted_query) return f"View {view_name} created successfully." @@ -289,9 +280,7 @@ def mock_create_view( def mock_get_query( query_id: str, - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> Collection[Collection[str]]: if query_id == "foo-bar-baz": raise DJQueryServiceClientEntityNotFound("Query foo-bar-baz not found.") @@ -353,7 +342,7 @@ def mock_get_query( @pytest_asyncio.fixture -async def client( # pylint: disable=too-many-statements +async def client( session: AsyncSession, settings_no_qs: Settings, ) -> AsyncGenerator[AsyncClient, None]: @@ -544,7 +533,7 @@ def compare_parse_trees(tree1, tree2): Recursively compare two ANTLR parse trees for equality. """ # Check if the node types are the same - if type(tree1) is not type(tree2): # pylint: disable=unidiomatic-typecheck + if type(tree1) is not type(tree2): return False # Check if the node texts are the same @@ -591,7 +580,7 @@ def compare_query_strings_fixture(): @pytest_asyncio.fixture -async def client_with_query_service_example_loader( # pylint: disable=too-many-statements +async def client_with_query_service_example_loader( session: AsyncSession, settings: Settings, query_service_client: QueryServiceClient, @@ -603,7 +592,7 @@ async def client_with_query_service_example_loader( # pylint: disable=too-many- """ def get_query_service_client_override( - request: Request = None, # pylint: disable=unused-argument + request: Request = None, ) -> QueryServiceClient: return query_service_client @@ -643,7 +632,7 @@ def _load_examples(examples_to_load: Optional[List[str]] = None): @pytest_asyncio.fixture -async def client_with_query_service( # pylint: disable=too-many-statements +async def client_with_query_service( client_with_query_service_example_loader: Callable[ [Optional[List[str]]], AsyncClient, @@ -711,10 +700,10 @@ async def _load_examples(examples_to_load: Optional[List[str]] = None): @pytest_asyncio.fixture(scope="module") -async def module__client( # pylint: disable=too-many-statements +async def module__client( module__session: AsyncSession, module__settings: Settings, - module__query_service_client: QueryServiceClient, # pylint: disable=unused-argument + module__query_service_client: QueryServiceClient, module_mocker: MockerFixture, ) -> AsyncGenerator[AsyncClient, None]: """ @@ -730,7 +719,7 @@ async def module__client( # pylint: disable=too-many-statements await module__session.execute(statement) def get_query_service_client_override( - request: Request = None, # pylint: disable=unused-argument + request: Request = None, ) -> QueryServiceClient: return module__query_service_client @@ -933,7 +922,7 @@ def module__postgres_container(request) -> PostgresContainer: @pytest.fixture(scope="module") def module__query_service_client( module_mocker: MockerFixture, - duckdb_conn: duckdb.DuckDBPyConnection, # pylint: disable=c-extension-no-member + duckdb_conn: duckdb.DuckDBPyConnection, ) -> Iterator[QueryServiceClient]: """ Custom settings for unit tests. @@ -944,10 +933,8 @@ def mock_get_columns_for_table( catalog: str, schema: str, table: str, - engine: Optional[Engine] = None, # pylint: disable=unused-argument - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + engine: Optional[Engine] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> List[Column]: return COLUMN_MAPPINGS[f"{catalog}.{schema}.{table}"] @@ -959,9 +946,7 @@ def mock_get_columns_for_table( def mock_submit_query( query_create: QueryCreate, - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> QueryWithResults: result = duckdb_conn.sql(query_create.submitted_query) columns = [ @@ -991,9 +976,7 @@ def mock_submit_query( def mock_create_view( view_name: str, query_create: QueryCreate, - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> str: duckdb_conn.sql(query_create.submitted_query) return f"View {view_name} created successfully." @@ -1006,9 +989,7 @@ def mock_create_view( def mock_get_query( query_id: str, - request_headers: Optional[ # pylint: disable=unused-argument - Dict[str, str] - ] = None, + request_headers: Optional[Dict[str, str]] = None, ) -> Collection[Collection[str]]: if query_id == "foo-bar-baz": raise DJQueryServiceClientEntityNotFound("Query foo-bar-baz not found.") diff --git a/datajunction-server/tests/construction/build_v2_test.py b/datajunction-server/tests/construction/build_v2_test.py index e47280d51..1909b6272 100644 --- a/datajunction-server/tests/construction/build_v2_test.py +++ b/datajunction-server/tests/construction/build_v2_test.py @@ -1,4 +1,3 @@ -# pylint: disable=redefined-outer-name,too-many-lines """Tests for building nodes""" from typing import List, Tuple @@ -570,7 +569,7 @@ async def test_dimension_join_path( session: AsyncSession, events: Node, events_agg: Node, - events_agg_devices_link: Node, # pylint: disable=unused-argument + events_agg_devices_link: Node, ): """ Test finding a join path between the dimension attribute and the node. @@ -718,8 +717,8 @@ async def test_build_source_node_with_direct_filter( async def test_build_source_with_pushdown_filters( session: AsyncSession, events: Node, - devices: Node, # pylint: disable=unused-argument - events_devices_link: DimensionLink, # pylint: disable=unused-argument + devices: Node, + events_devices_link: DimensionLink, ): """ Test building a source node with a dimension attribute filter that can be @@ -775,8 +774,8 @@ async def test_build_source_with_pushdown_filters( async def test_build_source_with_join_filters( session: AsyncSession, events: Node, - devices: Node, # pylint: disable=unused-argument - events_devices_link: DimensionLink, # pylint: disable=unused-argument + devices: Node, + events_devices_link: DimensionLink, ): """ Test building a source node with a dimension attribute filter that @@ -875,9 +874,9 @@ async def test_build_dimension_node( @pytest.mark.asyncio async def test_build_dimension_node_with_direct_and_pushdown_filter( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, devices: Node, - events_agg_devices_link: DimensionLink, # pylint: disable=unused-argument + events_agg_devices_link: DimensionLink, ): """ Test building a dimension node with a direct filter and a pushdown filter (the result @@ -922,11 +921,11 @@ async def test_build_dimension_node_with_direct_and_pushdown_filter( @pytest.mark.asyncio async def test_build_transform_with_pushdown_dimensions_filters( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - devices: Node, # pylint: disable=unused-argument - events_agg_devices_link: DimensionLink, # pylint: disable=unused-argument - manufacturers_dim: Node, # pylint: disable=unused-argument + devices: Node, + events_agg_devices_link: DimensionLink, + manufacturers_dim: Node, ): """ Test building a transform node with filters and dimensions that can be pushed down @@ -973,12 +972,12 @@ async def test_build_transform_with_pushdown_dimensions_filters( @pytest.mark.asyncio async def test_build_transform_with_deeper_pushdown_dimensions_filters( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - events_devices_link: DimensionLink, # pylint: disable=unused-argument - devices: Node, # pylint: disable=unused-argument - events_agg_devices_link: DimensionLink, # pylint: disable=unused-argument - manufacturers_dim: Node, # pylint: disable=unused-argument + events_devices_link: DimensionLink, + devices: Node, + events_agg_devices_link: DimensionLink, + manufacturers_dim: Node, ): """ Test building a transform node with filters and dimensions that can be pushed down @@ -1035,12 +1034,12 @@ async def test_build_transform_with_deeper_pushdown_dimensions_filters( @pytest.mark.asyncio async def test_build_transform_w_cte_and_pushdown_dimensions_filters( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg_complex: Node, - events_devices_link: DimensionLink, # pylint: disable=unused-argument - devices: Node, # pylint: disable=unused-argument - events_agg_complex_devices_link: DimensionLink, # pylint: disable=unused-argument - manufacturers_dim: Node, # pylint: disable=unused-argument + events_devices_link: DimensionLink, + devices: Node, + events_agg_complex_devices_link: DimensionLink, + manufacturers_dim: Node, ): """ Test building a transform node that has CTEs in the node query, built with @@ -1106,11 +1105,11 @@ async def test_build_transform_w_cte_and_pushdown_dimensions_filters( @pytest.mark.asyncio async def test_build_transform_with_join_dimensions_filters( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - devices: Node, # pylint: disable=unused-argument - events_agg_devices_link: DimensionLink, # pylint: disable=unused-argument - manufacturers_dim: Node, # pylint: disable=unused-argument + devices: Node, + events_agg_devices_link: DimensionLink, + manufacturers_dim: Node, ): """ Test building a transform node with filters and dimensions that require a join @@ -1169,12 +1168,12 @@ async def test_build_transform_with_join_dimensions_filters( @pytest.mark.asyncio async def test_build_transform_with_multijoin_dimensions_filters( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - devices: Node, # pylint: disable=unused-argument - events_agg_devices_link: DimensionLink, # pylint: disable=unused-argument - manufacturers_dim: Node, # pylint: disable=unused-argument - country_dim: Node, # pylint: disable=unused-argument + devices: Node, + events_agg_devices_link: DimensionLink, + manufacturers_dim: Node, + country_dim: Node, ): """ Test building a transform node with filters and dimensions that require @@ -1252,9 +1251,9 @@ async def test_build_transform_with_multijoin_dimensions_filters( @pytest.mark.asyncio async def test_build_fail_no_join_path_found( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - country_dim: Node, # pylint: disable=unused-argument + country_dim: Node, ): """ Test failed node building due to not being able to find a join path to the dimension @@ -1303,9 +1302,9 @@ async def test_build_fail_no_join_path_found( @pytest.mark.asyncio async def test_query_builder( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - country_dim: Node, # pylint: disable=unused-argument + country_dim: Node, ): """ Test failed node building due to not being able to find a join path to the dimension @@ -1328,24 +1327,24 @@ async def test_query_builder( ) assert query_builder.filters == ["shared.countries.region_name = 'APAC'"] assert query_builder.dimensions == ["shared.countries.region_name"] - assert query_builder._orderby == [ # pylint: disable=protected-access + assert query_builder._orderby == [ "shared.countries.region_name DESC", "shared.countries.region_name ASC", ] - assert query_builder._limit == 100 # pylint: disable=protected-access + assert query_builder._limit == 100 assert not query_builder.include_dimensions_in_groupby @pytest.mark.asyncio async def test_build_transform_sql_without_materialized_tables( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - devices: Node, # pylint: disable=unused-argument - events_agg_devices_link: DimensionLink, # pylint: disable=unused-argument - manufacturers_dim: Node, # pylint: disable=unused-argument - country_dim: Node, # pylint: disable=unused-argument - events_agg_countries_link: DimensionLink, # pylint: disable=unused-argument + devices: Node, + events_agg_devices_link: DimensionLink, + manufacturers_dim: Node, + country_dim: Node, + events_agg_countries_link: DimensionLink, ): """ Test building a transform node with filters and dimensions that forces skipping the materialized @@ -1471,13 +1470,13 @@ async def test_build_transform_sql_without_materialized_tables( @pytest.mark.asyncio async def test_build_transform_with_multijoin_dimensions_with_extra_ctes( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - devices: Node, # pylint: disable=unused-argument - events_agg_devices_link: DimensionLink, # pylint: disable=unused-argument - manufacturers_dim: Node, # pylint: disable=unused-argument - country_dim: Node, # pylint: disable=unused-argument - events_agg_countries_link: DimensionLink, # pylint: disable=unused-argument + devices: Node, + events_agg_devices_link: DimensionLink, + manufacturers_dim: Node, + country_dim: Node, + events_agg_countries_link: DimensionLink, ): """ Test building a transform node with filters and dimensions that require @@ -1570,10 +1569,10 @@ async def test_build_transform_with_multijoin_dimensions_with_extra_ctes( @pytest.mark.asyncio async def test_build_with_source_filters( session: AsyncSession, - events: Node, # pylint: disable=unused-argument + events: Node, events_agg: Node, - date_dim: Node, # pylint: disable=unused-argument - events_agg_date_dim_link: DimensionLink, # pylint: disable=unused-argument + date_dim: Node, + events_agg_date_dim_link: DimensionLink, ): """ Test build node with filters on source diff --git a/datajunction-server/tests/construction/compile_test.py b/datajunction-server/tests/construction/compile_test.py index 4ffe5f145..fff1f69bd 100644 --- a/datajunction-server/tests/construction/compile_test.py +++ b/datajunction-server/tests/construction/compile_test.py @@ -2,7 +2,6 @@ Tests for compiling nodes """ -# pylint: disable=too-many-lines import pytest from sqlalchemy.ext.asyncio import AsyncSession diff --git a/datajunction-server/tests/construction/conftest.py b/datajunction-server/tests/construction/conftest.py index 9628c76d3..27320cec3 100644 --- a/datajunction-server/tests/construction/conftest.py +++ b/datajunction-server/tests/construction/conftest.py @@ -1,6 +1,4 @@ """fixtures for testing construction""" -# noqa: W191,E101 -# pylint: disable=line-too-long,too-many-statements from typing import Dict, List, Optional, Tuple @@ -194,7 +192,7 @@ def build_expectation() -> Dict[str, Dict[Optional[int], Tuple[bool, str]]]: @pytest_asyncio.fixture -async def construction_session( # pylint: disable=too-many-locals +async def construction_session( session: AsyncSession, current_user: User, ) -> AsyncSession: @@ -412,8 +410,8 @@ async def construction_session( # pylint: disable=too-many-locals Column(name="cnt", type=IntegerType(), order=0), ], required_dimensions=[ - comments_src.columns[0], # pylint: disable=E1136 - comments_src.columns[-1], # pylint: disable=E1136 + comments_src.columns[0], + comments_src.columns[-1], ], created_by_id=current_user.id, ) diff --git a/datajunction-server/tests/construction/exceptions_test.py b/datajunction-server/tests/construction/exceptions_test.py index 122da6aa8..0256fbddd 100644 --- a/datajunction-server/tests/construction/exceptions_test.py +++ b/datajunction-server/tests/construction/exceptions_test.py @@ -1,7 +1,6 @@ """ Tests for building nodes and extracting dependencies """ -# pylint: disable=too-many-lines import pytest @@ -14,7 +13,7 @@ def test_compound_build_exception(): Test raising a CompoundBuildException """ CompoundBuildException().reset() - CompoundBuildException().set_raise(False) # pylint: disable=protected-access + CompoundBuildException().set_raise(False) CompoundBuildException().append( error=DJError( code=ErrorCode.INVALID_SQL_QUERY, @@ -36,7 +35,7 @@ def test_raise_compound_build_exception(): Test raising a CompoundBuildException """ CompoundBuildException().reset() - CompoundBuildException().set_raise(True) # pylint: disable=protected-access + CompoundBuildException().set_raise(True) with pytest.raises(DJException) as exc_info: CompoundBuildException().append( error=DJError( diff --git a/datajunction-server/tests/construction/inference_test.py b/datajunction-server/tests/construction/inference_test.py index 15fc48725..03b2e1b63 100644 --- a/datajunction-server/tests/construction/inference_test.py +++ b/datajunction-server/tests/construction/inference_test.py @@ -1,6 +1,5 @@ """Test type inference.""" -# pylint: disable=W0621,C0325 import pytest from sqlalchemy.ext.asyncio import AsyncSession @@ -69,7 +68,7 @@ def test_raise_on_invalid_infer_binary_op(): Test raising when trying to infer types from an invalid binary op """ with pytest.raises(DJParseException) as exc_info: - ast.BinaryOp( # pylint: disable=expression-not-assigned + ast.BinaryOp( op=ast.BinaryOpKind.Modulo, left=ast.String(value="foo"), right=ast.String(value="bar"), @@ -123,7 +122,7 @@ def test_raising_when_table_has_no_dj_node(): col = ast.Column(ast.Name("status"), _table=table) with pytest.raises(DJParseException) as exc_info: - col.type # pylint: disable=pointless-statement + col.type assert ("Cannot resolve type of column orders.status") in str(exc_info.value) @@ -133,7 +132,7 @@ def test_raising_when_select_has_multiple_expressions_in_projection(): Test raising when a select has more than one in projection """ with pytest.raises(DJParseException) as exc_info: - parse("select 1, 2").select.type # pylint: disable=expression-not-assigned + parse("select 1, 2").select.type assert ("single expression in its projection") in str(exc_info.value) @@ -143,7 +142,7 @@ def test_raising_when_between_different_types(): Test raising when a between has multiple types """ with pytest.raises(DJParseException) as exc_info: - parse( # pylint: disable=expression-not-assigned + parse( "select 1 between 'hello' and TRUE", ).select.type @@ -155,7 +154,7 @@ def test_raising_when_unop_bad_type(): Test raising when a unop gets a bad type """ with pytest.raises(DJParseException) as exc_info: - parse( # pylint: disable=expression-not-assigned + parse( "select not 'hello'", ).select.type @@ -169,7 +168,7 @@ def test_raising_when_expression_has_no_parent(): col = ast.Column(ast.Name("status"), _table=None) with pytest.raises(DJParseException) as exc_info: - col.type # pylint: disable=pointless-statement + col.type assert "Cannot resolve type of column status that has no parent" in str( exc_info.value, @@ -331,7 +330,7 @@ async def test_infer_bad_case_types(construction_session: AsyncSession): exception=DJException(), ) await query.compile(ctx) - [ # pylint: disable=pointless-statement + [ exp.type # type: ignore for exp in query.select.projection ] @@ -505,7 +504,7 @@ async def test_infer_types_array_map(construction_session: AsyncSession): ctx = CompileContext(session=construction_session, exception=exc) await query.compile(ctx) with pytest.raises(DJParseException) as exc_info: - query.select.projection[0].type # type: ignore # pylint: disable=pointless-statement + query.select.projection[0].type # type: ignore assert "Multiple types int, string passed to array" in str(exc_info) @@ -528,7 +527,7 @@ async def test_infer_types_if(construction_session: AsyncSession): await query.compile(ctx) assert query.select.projection[0].type == IntegerType() # type: ignore with pytest.raises(DJException) as exc_info: - query.select.projection[1].type # type: ignore # pylint: disable=pointless-statement + query.select.projection[1].type # type: ignore assert ( "The then result and else result must match in type! Got string and int" in str(exc_info) diff --git a/datajunction-server/tests/construction/utils_test.py b/datajunction-server/tests/construction/utils_test.py index 3b3995e2d..a4cd5a3dc 100644 --- a/datajunction-server/tests/construction/utils_test.py +++ b/datajunction-server/tests/construction/utils_test.py @@ -2,7 +2,6 @@ Tests for building nodes and extracting dependencies """ -# pylint: disable=too-many-lines import pytest from sqlalchemy.ext.asyncio import AsyncSession @@ -37,10 +36,7 @@ async def test_get_dj_node_raise_unknown_node_exception(session: AsyncSession): # test that the event_type raises because it's a dimension and not a transform await get_dj_node(session, "event_type", kinds={NodeType.TRANSFORM}) - assert ( - "No node `event_type` exists of kind transform" # pylint: disable=C0301 - in str(exc_info.value) - ) + assert "No node `event_type` exists of kind transform" in str(exc_info.value) # test that the event_type raises because it's a dimension and not a transform with pytest.raises(DJErrorException) as exc_info: diff --git a/datajunction-server/tests/examples.py b/datajunction-server/tests/examples.py index da6f0e165..15582908e 100644 --- a/datajunction-server/tests/examples.py +++ b/datajunction-server/tests/examples.py @@ -1,5 +1,3 @@ -# pylint: disable=too-many-lines,C0301 - """ Post requests for all example entities """ diff --git a/datajunction-server/tests/migrations_test.py b/datajunction-server/tests/migrations_test.py index 40a8b31e5..868bc08c5 100644 --- a/datajunction-server/tests/migrations_test.py +++ b/datajunction-server/tests/migrations_test.py @@ -6,6 +6,7 @@ from alembic.runtime.environment import EnvironmentContext from alembic.runtime.migration import MigrationContext from alembic.script import ScriptDirectory +from httpx import AsyncClient from sqlalchemy import create_engine from sqlalchemy.engine.base import Connection from testcontainers.postgres import PostgresContainer @@ -28,7 +29,7 @@ def connection(postgres_container: PostgresContainer) -> Connection: transaction.rollback() -def test_migrations_are_current(connection): # pylint: disable=redefined-outer-name +def test_migrations_are_current(connection): """ Verify that the alembic migrations are in line with the models. """ @@ -41,7 +42,7 @@ def test_migrations_are_current(connection): # pylint: disable=redefined-outer- context = EnvironmentContext( config, script, - fn=lambda rev, _: script._upgrade_revs("head", rev), # pylint: disable=W0212 + fn=lambda rev, _: script._upgrade_revs("head", rev), ) context.configure(connection=connection) context.run_migrations() @@ -53,3 +54,18 @@ def test_migrations_are_current(connection): # pylint: disable=redefined-outer- ) diff = compare_metadata(migrations_state, target_metadata) assert diff == [], "The alembic migrations do not match the models." + + +@pytest.mark.asyncio +async def test_openapi_schema(client: AsyncClient): + """ + Fetch and validate the OpenAPI schema. + """ + response = await client.get("/openapi.json") + assert response.status_code == 200, "Failed to fetch OpenAPI schema" + + schema = response.json() + assert "openapi" in schema, "Missing 'openapi' version field" + assert "info" in schema, "Missing 'info' section" + assert "paths" in schema, "Missing 'paths' section" + assert "components" in schema, "Missing 'components' section" diff --git a/datajunction-server/tests/models/node_test.py b/datajunction-server/tests/models/node_test.py index 5d7a66ccb..1ab36079d 100644 --- a/datajunction-server/tests/models/node_test.py +++ b/datajunction-server/tests/models/node_test.py @@ -2,8 +2,6 @@ Tests for ``datajunction_server.models.node``. """ -# pylint: disable=use-implicit-booleaness-not-comparison - import pytest from sqlalchemy.ext.asyncio import AsyncSession @@ -317,5 +315,5 @@ def test_node_cursors() -> None: assert cursor.encode() == encoded_cursor decoded_cursor = NodeCursor.decode(encoded_cursor) - assert decoded_cursor.created_at == cursor.created_at # pylint: disable=no-member - assert decoded_cursor.id == cursor.id # pylint: disable=no-member + assert decoded_cursor.created_at == cursor.created_at + assert decoded_cursor.id == cursor.id diff --git a/datajunction-server/tests/service_clients_test.py b/datajunction-server/tests/service_clients_test.py index 5c5340fde..5fb79b6f6 100644 --- a/datajunction-server/tests/service_clients_test.py +++ b/datajunction-server/tests/service_clients_test.py @@ -90,7 +90,7 @@ def test_make_requests( ) -class TestQueryServiceClient: # pylint: disable=too-few-public-methods +class TestQueryServiceClient: """ Test using the query service client. """ diff --git a/datajunction-server/tests/sql/functions_test.py b/datajunction-server/tests/sql/functions_test.py index fa5da52c8..67c6f4bd4 100644 --- a/datajunction-server/tests/sql/functions_test.py +++ b/datajunction-server/tests/sql/functions_test.py @@ -1,4 +1,3 @@ -# pylint: disable=line-too-long,too-many-lines """ Tests for ``datajunction_server.sql.functions``. """ @@ -43,7 +42,7 @@ async def test_missing_functions() -> None: Test missing functions. """ with pytest.raises(DJNotImplementedException) as excinfo: - function_registry["INVALID_FUNCTION"] # pylint: disable=pointless-statement + function_registry["INVALID_FUNCTION"] assert ( str(excinfo.value) == "The function `INVALID_FUNCTION` hasn't been implemented " "in DJ yet. You can file an issue at https://github.com/" @@ -1809,7 +1808,7 @@ async def test_format_string_func(session: AsyncSession): assert query.select.projection[1].type == ct.StringType() # type: ignore -# TODO: Fix these two # pylint: disable=fixme +# TODO: Fix these two # @pytest.mark.asyncio # async def test_from_csv_func(session: AsyncSession): # """ @@ -3064,7 +3063,7 @@ async def test_octet_length_func(session: AsyncSession): async def test_overlay_func(session: AsyncSession): """ Test the `overlay` function - TODO: support syntax like: # pylint: disable=fixme + TODO: support syntax like: SELECT overlay(encode('Spark SQL', 'utf-8') PLACING encode('_', 'utf-8') FROM 6); """ query = parse("SELECT overlay('Hello World', 'J', 7)") diff --git a/datajunction-server/tests/sql/parsing/queries/tpcds/test_tpcds.py b/datajunction-server/tests/sql/parsing/queries/tpcds/test_tpcds.py index 3bf7e5f04..55c486f18 100644 --- a/datajunction-server/tests/sql/parsing/queries/tpcds/test_tpcds.py +++ b/datajunction-server/tests/sql/parsing/queries/tpcds/test_tpcds.py @@ -3,7 +3,6 @@ """ # mypy: ignore-errors -# pylint: skip-file from difflib import SequenceMatcher import pytest diff --git a/datajunction-server/tests/sql/parsing/test_ast.py b/datajunction-server/tests/sql/parsing/test_ast.py index fba7ea66a..9218601e8 100644 --- a/datajunction-server/tests/sql/parsing/test_ast.py +++ b/datajunction-server/tests/sql/parsing/test_ast.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines,too-many-statements """ testing ast Nodes and their methods """ @@ -18,7 +17,7 @@ @pytest.mark.asyncio async def test_ast_compile_table( session: AsyncSession, - client_with_roads: AsyncClient, # pylint: disable=unused-argument + client_with_roads: AsyncClient, ): """ Test compiling the primary table from a query @@ -31,7 +30,7 @@ async def test_ast_compile_table( await query.select.from_.relations[0].primary.compile(ctx) # type: ignore assert not exc.errors - node = query.select.from_.relations[ # type: ignore # pylint: disable=protected-access + node = query.select.from_.relations[ # type: ignore 0 ].primary._dj_node assert node @@ -83,7 +82,7 @@ async def test_ast_compile_table_missing_node(session: AsyncSession): @pytest.mark.asyncio async def test_ast_compile_query( session: AsyncSession, - client_with_roads: AsyncClient, # pylint: disable=unused-argument + client_with_roads: AsyncClient, ): """ Test compiling an entire query @@ -94,7 +93,7 @@ async def test_ast_compile_query( await query.compile(ctx) assert not exc.errors - node = query.select.from_.relations[ # type: ignore # pylint: disable=protected-access + node = query.select.from_.relations[ # type: ignore 0 ].primary._dj_node assert node @@ -104,7 +103,7 @@ async def test_ast_compile_query( @pytest.mark.asyncio async def test_ast_compile_query_missing_columns( session: AsyncSession, - client_with_roads: AsyncClient, # pylint: disable=unused-argument + client_with_roads: AsyncClient, ): """ Test compiling a query with missing columns @@ -122,7 +121,7 @@ async def test_ast_compile_query_missing_columns( in exc.errors[1].message ) - node = query.select.from_.relations[ # type: ignore # pylint: disable=protected-access + node = query.select.from_.relations[ # type: ignore 0 ].primary._dj_node assert node @@ -145,7 +144,7 @@ async def test_ast_compile_missing_references(session: AsyncSession): @pytest.mark.asyncio async def test_ast_compile_raise_on_ambiguous_column( session: AsyncSession, - client_with_basic: AsyncClient, # pylint: disable=unused-argument + client_with_basic: AsyncClient, ): """ Test raising on ambiguous column @@ -166,7 +165,7 @@ async def test_ast_compile_raise_on_ambiguous_column( @pytest.mark.asyncio async def test_ast_compile_having( session: AsyncSession, - client_with_dbt: AsyncClient, # pylint: disable=unused-argument + client_with_dbt: AsyncClient, ): """ Test using having @@ -180,7 +179,7 @@ async def test_ast_compile_having( ctx = ast.CompileContext(session=session, exception=exc) await query.compile(ctx) - node = query.select.from_.relations[0].primary._dj_node # type: ignore # pylint: disable=protected-access + node = query.select.from_.relations[0].primary._dj_node # type: ignore assert node assert node.name == "dbt.source.jaffle_shop.orders" diff --git a/datajunction-server/tests/transpilation_test.py b/datajunction-server/tests/transpilation_test.py index 5f92715a6..8c752f6da 100644 --- a/datajunction-server/tests/transpilation_test.py +++ b/datajunction-server/tests/transpilation_test.py @@ -10,7 +10,7 @@ from datajunction_server.transpilation import get_transpilation_plugin -class MockSettings: # pylint: disable=too-few-public-methods +class MockSettings: """ Mock settings object """ diff --git a/datajunction-server/tests/utils_test.py b/datajunction-server/tests/utils_test.py index 9ec24b71a..d15bf2004 100644 --- a/datajunction-server/tests/utils_test.py +++ b/datajunction-server/tests/utils_test.py @@ -59,7 +59,7 @@ def test_get_settings(mocker: MockerFixture) -> None: Test ``get_settings``. """ mocker.patch("datajunction_server.utils.load_dotenv") - Settings = mocker.patch( # pylint: disable=invalid-name, redefined-outer-name + Settings = mocker.patch( "datajunction_server.utils.Settings", )