From 80f4de159f690fb4554395380ca8554018d11203 Mon Sep 17 00:00:00 2001 From: Mathias Lohne Date: Fri, 13 Dec 2024 21:00:58 +0100 Subject: [PATCH] Clean up type hints in tests (#410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many tests were completely untyped, many had incorrect typing. This doesn't fix everything, there's still a lot of type errors in the tests, but it is an improvement. I'll maybe make a few more PRs cleaning up a bit more in the tests, but some of it is also fine to leave as it might get replaced altogether in the next major release. Co-authored-by: Ozan Göktan <72358629+ozangoktan@users.noreply.github.com> --- tests/conftest.py | 12 ++-- .../test_assets_integration.py | 3 +- .../test_configtools_integration.py | 4 +- .../test_file_integration.py | 16 ++--- .../tests_integration/test_raw_integration.py | 3 +- .../test_time_series_integration.py | 12 ++-- tests/tests_unit/test_base.py | 12 ++-- tests/tests_unit/test_cdf_upload_queues.py | 62 ++++++++++--------- tests/tests_unit/test_configtools.py | 40 ++++++------ tests/tests_unit/test_extraction_pipelines.py | 16 ++--- tests/tests_unit/test_metrics.py | 58 +++++++++-------- tests/tests_unit/test_statestore.py | 34 +++++----- tests/tests_unit/test_uploader_extractor.py | 15 +++-- tests/tests_unit/test_util.py | 49 +++++++-------- 14 files changed, 168 insertions(+), 168 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 273b0bf5..1ebded91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import os from dataclasses import dataclass from enum import Enum -from typing import Generator, List, Optional, Tuple +from typing import Generator import pytest @@ -27,16 +27,16 @@ class ETestType(Enum): @dataclass class ParamTest: test_type: ETestType - external_ids: Optional[List[str]] = None - database_name: Optional[str] = None - table_name: Optional[str] = None - space: Optional[str] = None + external_ids: list[str] | None = None + database_name: str | None = None + table_name: str | None = None + space: str | None = None @pytest.fixture def set_upload_test( set_test_parameters: ParamTest, set_client: CogniteClient -) -> Generator[Tuple[CogniteClient, ParamTest], None, None]: +) -> Generator[tuple[CogniteClient, ParamTest], None, None]: client = set_client test_parameter = set_test_parameters clean_test(client, test_parameter) diff --git a/tests/tests_integration/test_assets_integration.py b/tests/tests_integration/test_assets_integration.py index 911a489d..c3c59cdb 100644 --- a/tests/tests_integration/test_assets_integration.py +++ b/tests/tests_integration/test_assets_integration.py @@ -14,7 +14,6 @@ import os import random -from typing import Tuple import pytest @@ -37,7 +36,7 @@ def set_test_parameters() -> ParamTest: @pytest.mark.parametrize("functions_runtime", ["true", "false"]) -def test_assets_upload_queue_upsert(set_upload_test: Tuple[CogniteClient, ParamTest], functions_runtime: str): +def test_assets_upload_queue_upsert(set_upload_test: tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: os.environ["COGNITE_FUNCTION_RUNTIME"] = functions_runtime client, test_parameter = set_upload_test queue = AssetUploadQueue(cdf_client=client) diff --git a/tests/tests_integration/test_configtools_integration.py b/tests/tests_integration/test_configtools_integration.py index cc5335c3..30c827f5 100644 --- a/tests/tests_integration/test_configtools_integration.py +++ b/tests/tests_integration/test_configtools_integration.py @@ -28,7 +28,7 @@ class DummyConfig(BaseConfig): dummy_secret: str -def test_dataset_resolve(set_client: CogniteClient): +def test_dataset_resolve(set_client: CogniteClient) -> None: client = set_client data_set_name: str = "Extractor Utils Test Data Set" data_set_extid: str = "extractorUtils-testdataset" @@ -92,7 +92,7 @@ def test_dataset_resolve(set_client: CogniteClient): assert config2.cognite.get_data_set(client).name == data_set_name -def test_keyvault_and_remote(set_client: CogniteClient): +def test_keyvault_and_remote(set_client: CogniteClient) -> None: # Set up extraction pipeline data_set_name: str = "Extractor Utils Test Data Set" data_set_extid: str = "extractorUtils-testdataset" diff --git a/tests/tests_integration/test_file_integration.py b/tests/tests_integration/test_file_integration.py index 92fc68d1..8d971b31 100644 --- a/tests/tests_integration/test_file_integration.py +++ b/tests/tests_integration/test_file_integration.py @@ -17,7 +17,7 @@ import pathlib import random import time -from typing import Callable, Optional, Tuple +from typing import Callable import pytest @@ -53,7 +53,7 @@ def set_test_parameters() -> ParamTest: def await_is_uploaded_status( - client: CogniteClient, external_id: Optional[str] = None, instance_id: Optional[NodeId] = None + client: CogniteClient, external_id: str | None = None, instance_id: NodeId | None = None ) -> None: for _ in range(10): if external_id is not None: @@ -68,7 +68,7 @@ def await_is_uploaded_status( @pytest.mark.parametrize("functions_runtime", ["true", "false"]) -def test_file_upload_queue(set_upload_test: Tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: +def test_file_upload_queue(set_upload_test: tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: os.environ["COGNITE_FUNCTION_RUNTIME"] = functions_runtime client, test_parameter = set_upload_test queue = FileUploadQueue(cdf_client=client, overwrite_existing=True, max_queue_size=2) @@ -145,7 +145,7 @@ def test_file_upload_queue(set_upload_test: Tuple[CogniteClient, ParamTest], fun @pytest.mark.parametrize("functions_runtime", ["true", "false"]) -def test_bytes_upload_queue(set_upload_test: Tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: +def test_bytes_upload_queue(set_upload_test: tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: os.environ["COGNITE_FUNCTION_RUNTIME"] = functions_runtime client, test_parameter = set_upload_test queue = BytesUploadQueue(cdf_client=client, overwrite_existing=True, max_queue_size=1) @@ -193,7 +193,7 @@ def test_bytes_upload_queue(set_upload_test: Tuple[CogniteClient, ParamTest], fu @pytest.mark.parametrize("functions_runtime", ["true", "false"]) -def test_big_file_upload_queue(set_upload_test: Tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: +def test_big_file_upload_queue(set_upload_test: tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: os.environ["COGNITE_FUNCTION_RUNTIME"] = functions_runtime client, test_parameter = set_upload_test queue = BytesUploadQueue(cdf_client=client, overwrite_existing=True, max_queue_size=1) @@ -228,7 +228,7 @@ def test_big_file_upload_queue(set_upload_test: Tuple[CogniteClient, ParamTest], assert len(bigfile2) == 10_000_000 -def test_big_file_stream(set_upload_test: Tuple[CogniteClient, ParamTest]) -> None: +def test_big_file_stream(set_upload_test: tuple[CogniteClient, ParamTest]) -> None: client, test_parameter = set_upload_test queue = IOFileUploadQueue(cdf_client=client, overwrite_existing=True, max_queue_size=1) queue.max_file_chunk_size = 6_000_000 @@ -238,7 +238,7 @@ def test_big_file_stream(set_upload_test: Tuple[CogniteClient, ParamTest]) -> No class BufferedReadWithLength(io.BufferedReader): def __init__( - self, raw: io.RawIOBase, buffer_size: int, len: int, on_close: Optional[Callable[[], None]] = None + self, raw: io.RawIOBase, buffer_size: int, len: int, on_close: Callable[[], None] | None = None ) -> None: super().__init__(raw, buffer_size) # Do not remove even if it appears to be unused. :P @@ -279,7 +279,7 @@ def read_file() -> BufferedReadWithLength: assert len(bigfile2) == 10_000_000 -def test_update_files(set_upload_test: Tuple[CogniteClient, ParamTest]) -> None: +def test_update_files(set_upload_test: tuple[CogniteClient, ParamTest]) -> None: client, test_parameter = set_upload_test queue = BytesUploadQueue(cdf_client=client, overwrite_existing=True, max_queue_size=1) diff --git a/tests/tests_integration/test_raw_integration.py b/tests/tests_integration/test_raw_integration.py index c70eb899..b06f9898 100644 --- a/tests/tests_integration/test_raw_integration.py +++ b/tests/tests_integration/test_raw_integration.py @@ -15,7 +15,6 @@ import os import random import time -from typing import Tuple import pytest @@ -35,7 +34,7 @@ def set_test_parameters() -> ParamTest: @pytest.mark.parametrize("functions_runtime", ["true", "false"]) -def test_raw_upload_queue(set_upload_test: Tuple[CogniteClient, ParamTest], functions_runtime: str): +def test_raw_upload_queue(set_upload_test: tuple[CogniteClient, ParamTest], functions_runtime: str) -> None: os.environ["COGNITE_FUNCTION_RUNTIME"] = functions_runtime client, test_parameter = set_upload_test queue = RawUploadQueue(cdf_client=client, max_queue_size=500) diff --git a/tests/tests_integration/test_time_series_integration.py b/tests/tests_integration/test_time_series_integration.py index c29c7349..804fda78 100644 --- a/tests/tests_integration/test_time_series_integration.py +++ b/tests/tests_integration/test_time_series_integration.py @@ -16,7 +16,7 @@ import string import time from datetime import datetime, timezone -from typing import Tuple +from typing import Any import pytest @@ -38,7 +38,7 @@ def set_test_parameters() -> ParamTest: return test_parameter -def test_time_series_upload_queue1(set_upload_test: Tuple[CogniteClient, ParamTest]) -> None: +def test_time_series_upload_queue1(set_upload_test: tuple[CogniteClient, ParamTest]) -> None: client, test_parameter = set_upload_test created = client.time_series.create( [ @@ -49,7 +49,7 @@ def test_time_series_upload_queue1(set_upload_test: Tuple[CogniteClient, ParamTe last_point = {"timestamp": 0} - def store_latest(points): + def store_latest(points: Any) -> None: last_point["timestamp"] = max(last_point["timestamp"], *[ts["datapoints"][-1][0] for ts in points]) queue = TimeSeriesUploadQueue(cdf_client=client, post_upload_function=store_latest, max_upload_interval=1) @@ -82,7 +82,7 @@ def store_latest(points): queue.stop() -def test_time_series_upload_queue2(set_upload_test: Tuple[CogniteClient, ParamTest]) -> None: +def test_time_series_upload_queue2(set_upload_test: tuple[CogniteClient, ParamTest]) -> None: client, test_parameter = set_upload_test client.time_series.create(TimeSeries(external_id=test_parameter.external_ids[0])) @@ -109,7 +109,7 @@ def test_time_series_upload_queue2(set_upload_test: Tuple[CogniteClient, ParamTe queue.stop() -def test_time_series_upload_queue_create_missing(set_upload_test: Tuple[CogniteClient, ParamTest]) -> None: +def test_time_series_upload_queue_create_missing(set_upload_test: tuple[CogniteClient, ParamTest]) -> None: client, test_parameter = set_upload_test queue = TimeSeriesUploadQueue(cdf_client=client, create_missing=True) @@ -144,7 +144,7 @@ def test_time_series_upload_queue_create_missing(set_upload_test: Tuple[CogniteC queue.stop() -def test_time_seires_with_status(set_upload_test: Tuple[CogniteClient, ParamTest]) -> None: +def test_time_seires_with_status(set_upload_test: tuple[CogniteClient, ParamTest]) -> None: client, test_parameter = set_upload_test queue = TimeSeriesUploadQueue(cdf_client=client, create_missing=True) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index f745153e..34522305 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass, field +from typing import Callable from unittest.mock import patch import pytest +from cognite.client import CogniteClient from cognite.extractorutils import Extractor from cognite.extractorutils.configtools import BaseConfig, StateStoreConfig from cognite.extractorutils.statestore import LocalStateStore, NoStateStore @@ -42,13 +44,13 @@ class ConfigWithoutStates(BaseConfig): source: SourceConfig -def test_load_config(): +def test_load_config() -> None: e1 = Extractor(name="my_extractor1", description="description", config_class=ConfigWithStates) e1._initial_load_config("tests/tests_unit/dummyconfig.yaml") assert isinstance(e1.config, ConfigWithStates) -def test_load_config_keyvault(): +def test_load_config_keyvault() -> None: e7 = Extractor(name="my_extractor7", description="description", config_class=ConfigWithoutStates) e7._initial_load_config("tests/tests_unit/dummyconfig_keyvault.yaml") @@ -58,7 +60,7 @@ def test_load_config_keyvault(): @patch("cognite.client.CogniteClient") -def test_load_state_store(get_client_mock): +def test_load_state_store(get_client_mock: Callable[[], CogniteClient]) -> None: e2 = Extractor(name="my_extractor2", description="description", config_class=ConfigWithStates) e2._initial_load_config("tests/tests_unit/dummyconfig.yaml") e2.cognite_client = get_client_mock() @@ -95,7 +97,7 @@ def test_load_state_store(get_client_mock): @pytest.mark.order(1) -def test_config_getter(): +def test_config_getter() -> None: with pytest.raises(ValueError): Extractor.get_current_config() @@ -110,7 +112,7 @@ def test_config_getter(): @pytest.mark.order(2) -def test_state_store_getter(): +def test_state_store_getter() -> None: with pytest.raises(ValueError): Extractor.get_current_statestore() diff --git a/tests/tests_unit/test_cdf_upload_queues.py b/tests/tests_unit/test_cdf_upload_queues.py index dc1ee26e..cb94ff50 100644 --- a/tests/tests_unit/test_cdf_upload_queues.py +++ b/tests/tests_unit/test_cdf_upload_queues.py @@ -15,11 +15,12 @@ import datetime import math import time -from unittest.mock import patch +from io import BytesIO +from typing import Any +from unittest.mock import Mock, patch from httpx import URL, Request -from cognite.client import CogniteClient from cognite.client.data_classes import Event, Row from cognite.extractorutils.uploader import ( EventUploadQueue, @@ -31,8 +32,8 @@ @patch("cognite.client.CogniteClient") -def test_raw_uploader1(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_raw_uploader1(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() queue = RawUploadQueue(client) @@ -53,12 +54,12 @@ def test_raw_uploader1(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_raw_uploader2(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_raw_uploader2(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() post_upload_test = {"value": False} - def post(x): + def post(_x: Any) -> None: post_upload_test["value"] = True queue = RawUploadQueue(client, post_upload_function=post, max_queue_size=2) @@ -73,8 +74,8 @@ def post(x): @patch("cognite.client.CogniteClient") -def test_ts_uploader1(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_ts_uploader1(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() queue = TimeSeriesUploadQueue(client) @@ -97,12 +98,12 @@ def test_ts_uploader1(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_ts_uploader2(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_ts_uploader2(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() post_upload_test = {"value": False} - def post(x): + def post(_x: Any) -> None: post_upload_test["value"] = True queue = TimeSeriesUploadQueue(client, max_upload_interval=2, post_upload_function=post) @@ -130,12 +131,12 @@ def post(x): @patch("cognite.client.CogniteClient") -def test_ts_uploader_discard(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_ts_uploader_discard(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() post_upload_test = {"value": False} - def post(x): + def post(_x: Any) -> None: post_upload_test["value"] = True queue = TimeSeriesUploadQueue(client, max_upload_interval=2, post_upload_function=post) @@ -167,8 +168,8 @@ def post(x): @patch("cognite.client.CogniteClient") -def test_event_uploader1(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_event_uploader1(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() queue = EventUploadQueue(client) @@ -184,12 +185,12 @@ def test_event_uploader1(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_event_uploader2(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_event_uploader2(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() post_upload_test = {"value": False} - def post(x): + def post(_x: Any) -> None: post_upload_test["value"] = True queue = EventUploadQueue(client, max_upload_interval=2, post_upload_function=post) @@ -210,12 +211,12 @@ def post(x): @patch("cognite.client.CogniteClient") -def test_sequence_uploader1(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_sequence_uploader1(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() post_upload_test = {"value": 0, "rows": 0} - def post(x): + def post(x: Any) -> None: post_upload_test["value"] += 1 post_upload_test["rows"] += sum([len(e.values) for e in x]) @@ -243,12 +244,12 @@ def post(x): @patch("cognite.client.CogniteClient") -def test_sequence_uploader2(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_sequence_uploader2(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() post_upload_test = {"value": 0, "rows": 0} - def post(x): + def post(x: Any) -> None: post_upload_test["value"] += 1 post_upload_test["rows"] += sum([len(e.values) for e in x]) @@ -276,9 +277,9 @@ def post(x): @patch("cognite.client.CogniteClient") -def test_mock_private_link_upload(MockCogniteClient): +def test_mock_private_link_upload(MockCogniteClient: Mock) -> None: # mocking privatelink behavior, testing the URL swap - client: CogniteClient = MockCogniteClient() + client = MockCogniteClient() base_url_str = "https://qweasd-666.gremiocampeao.cognitedata.com" base_url = URL(base_url_str) @@ -288,8 +289,9 @@ def test_mock_private_link_upload(MockCogniteClient): mock_download_url = "https://restricted-api.gremiocampeao.cognitedata.com/downloadUrl/myfile" - mock_stream = "Até a pé nós iremos, para o que der e vier, mas o certo, mas o certo é que nós estaremos, com o Grêmio onde o Grêmio estiver".encode() + bytes_ = "Até a pé nós iremos, para o que der e vier, mas o certo, mas o certo é que nós estaremos, com o Grêmio onde o Grêmio estiver".encode() + mock_stream = BytesIO(bytes_) - response: Request = queue._get_file_upload_request(mock_download_url, mock_stream, len(mock_stream)) + response: Request = queue._get_file_upload_request(mock_download_url, mock_stream, len(bytes_)) assert response.url.netloc == base_url.netloc diff --git a/tests/tests_unit/test_configtools.py b/tests/tests_unit/test_configtools.py index fa7d3fb2..5157a045 100644 --- a/tests/tests_unit/test_configtools.py +++ b/tests/tests_unit/test_configtools.py @@ -65,7 +65,7 @@ class SimpleStringConfig: string_field: str -def test_ensure_snake_case(): +def test_ensure_snake_case() -> None: snake_dict = { "test_key": "testValue", "another_key": "another-value", @@ -93,7 +93,7 @@ def test_ensure_snake_case(): assert snake_dict == _to_snake_case(pascal_dict, "pascal") -def test_read_cognite_config(): +def test_read_cognite_config() -> None: config_raw = """ # CDF project (also known as tenant name) project: tenant-name @@ -124,7 +124,7 @@ def test_read_cognite_config(): assert client.config.client_name == "test-client" -def test_read_base_config(): +def test_read_base_config() -> None: config_raw = """ version: "1" @@ -166,7 +166,7 @@ def test_read_base_config(): assert config.logger.file is None -def test_read_invalid_missing_fields(): +def test_read_invalid_missing_fields() -> None: # missing project config_raw = """ # How to label uploaded data in CDF @@ -177,7 +177,7 @@ def test_read_invalid_missing_fields(): load_yaml(config_raw, CogniteConfig) -def test_read_invalid_extra_fields(): +def test_read_invalid_extra_fields() -> None: config_raw = """ # CDF project (also known as tenant name) project: tenant-name @@ -200,7 +200,7 @@ def test_read_invalid_extra_fields(): load_yaml(config_raw, CogniteConfig) -def test_read_invalid_wrong_type(): +def test_read_invalid_wrong_type() -> None: config_raw = """ # CDF project (also known as tenant name) project: 1234 @@ -220,7 +220,7 @@ def test_read_invalid_wrong_type(): load_yaml(config_raw, CogniteConfig) -def test_get_cognite_client_from_aad(): +def test_get_cognite_client_from_aad() -> None: config_raw = """ idp-authentication: tenant: foo @@ -242,7 +242,7 @@ def test_get_cognite_client_from_aad(): assert isinstance(cdf, CogniteClient) -def test_read_boolean_casting(): +def test_read_boolean_casting() -> None: os.environ["TRUE_FLAG"] = "true" os.environ["FALSE_FLAG"] = "FALSE" os.environ["STR_VAL"] = "TeST" @@ -264,7 +264,7 @@ def test_read_boolean_casting(): assert config.yet_another_string_field == "TeST" -def test_read_invalid_boolean_casting(): +def test_read_invalid_boolean_casting() -> None: os.environ["TRUE_FLAG"] = "true" os.environ["FALSE_FLAG"] = "FALSE" os.environ["INVALID_FLAG"] = "TEST" @@ -281,7 +281,7 @@ def test_read_invalid_boolean_casting(): load_yaml(config, CastingClass) -def test_read_relative_path(): +def test_read_relative_path() -> None: config = """ boolean-field: true another-boolean-field: false @@ -295,7 +295,7 @@ def test_read_relative_path(): assert config.path_field.name == "file.txt" -def test_read_absolute_path(): +def test_read_absolute_path() -> None: config = """ boolean-field: true another-boolean-field: false @@ -309,7 +309,7 @@ def test_read_absolute_path(): assert config.path_field.name == "file.txt" -def test_parse_time_interval(): +def test_parse_time_interval() -> None: assert TimeIntervalConfig("54").seconds == 54 assert TimeIntervalConfig("54s").seconds == 54 assert TimeIntervalConfig("120s").seconds == 120 @@ -320,7 +320,7 @@ def test_parse_time_interval(): assert TimeIntervalConfig("1h").minutes == pytest.approx(60) -def test_parse_file_size(): +def test_parse_file_size() -> None: assert FileSizeConfig("154584").bytes == 154584 assert FileSizeConfig("1kB").bytes == 1000 assert FileSizeConfig("25MB").bytes == 25_000_000 @@ -334,7 +334,7 @@ def test_parse_file_size(): assert FileSizeConfig("14.5 mb").kilobytes == pytest.approx(14_500) -def test_multiple_logging_console(): +def test_multiple_logging_console() -> None: config_file = """ logger: console: @@ -364,7 +364,7 @@ def test_multiple_logging_console(): logger.handlers.clear() -def test_multiple_logging_file(): +def test_multiple_logging_file() -> None: config_file_1 = """ logger: file: @@ -414,7 +414,7 @@ def test_multiple_logging_file(): logger.handlers.clear() -def test_dump_and_reload_config(): +def test_dump_and_reload_config() -> None: # Verify that dumping and reloading a config file doesn't fail due to _file_hash config = BaseConfig( type=None, @@ -443,7 +443,7 @@ def test_dump_and_reload_config(): load_yaml(config_file, BaseConfig) -def test_env_substitution(): +def test_env_substitution() -> None: os.environ["STRING_VALUE"] = "heyo" config_file1 = "string-field: ${STRING_VALUE}" @@ -472,7 +472,7 @@ def test_env_substitution(): assert config5.string_field == "veryheyocrowded" -def test_env_substitution_remote_check(): +def test_env_substitution_remote_check() -> None: os.environ["STRING_VALUE"] = "test" resolver = ConfigResolver("some-path.yml", BaseConfig) @@ -492,7 +492,7 @@ def test_env_substitution_remote_check(): assert resolver.is_remote -def test_cognite_validation(): +def test_cognite_validation() -> None: conf = CogniteConfig(project="", idp_authentication=AuthenticatorConfig(client_id="", scopes=[])) with pytest.raises(InvalidConfigError) as e: conf.get_cognite_client("client-name") @@ -579,7 +579,7 @@ def test_ignore_pattern() -> None: IgnorePattern("g*i", [RegExpFlag.IC], [RegExpFlag.IC]) -def test_castable_int_parsing(monkeypatch): +def test_castable_int_parsing(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("PORT_NUMBER", "8080") config = """ diff --git a/tests/tests_unit/test_extraction_pipelines.py b/tests/tests_unit/test_extraction_pipelines.py index a24485ad..0726c548 100644 --- a/tests/tests_unit/test_extraction_pipelines.py +++ b/tests/tests_unit/test_extraction_pipelines.py @@ -21,12 +21,12 @@ with monkeypatch_cognite_client() as m_client: - def test_work_as_expected(): + def test_work_as_expected() -> None: @add_extraction_pipeline( extraction_pipeline_ext_id="1", cognite_client=m_client, ) - def test_success(): + def test_success() -> None: print("Starting function 'test_work_as_expected'") print("Stopping function 'test_work_as_expected'") @@ -38,12 +38,12 @@ def test_success(): with monkeypatch_cognite_client() as m2_client: - def test_raise_error(): + def test_raise_error() -> None: @add_extraction_pipeline( extraction_pipeline_ext_id="2", cognite_client=m2_client, ) - def test_failure(): + def test_failure() -> None: print("Starting function 'test_raise_error'") raise Exception("Testing exceptions") print("Stopping function 'test_raise_error'") @@ -54,9 +54,9 @@ def test_failure(): with monkeypatch_cognite_client() as m3_client: - def test_2_heartbeats(): + def test_2_heartbeats() -> None: @add_extraction_pipeline(extraction_pipeline_ext_id="3", cognite_client=m3_client, heartbeat_waiting_time=1) - def test_success_2(): + def test_success_2() -> None: print("Starting function 'test_2_heartbeats'") time.sleep(1.5) print("Stopping function 'test_2_heartbeats'") @@ -65,7 +65,3 @@ def test_success_2(): print(f"{m3_client.extraction_pipelines.runs.create.call_count=}") assert m3_client.extraction_pipelines.runs.create.call_count == 3 - - -if __name__ == "__main__": - pytest.main(verbosity=2) diff --git a/tests/tests_unit/test_metrics.py b/tests/tests_unit/test_metrics.py index 2a766075..362d188e 100644 --- a/tests/tests_unit/test_metrics.py +++ b/tests/tests_unit/test_metrics.py @@ -13,6 +13,7 @@ # limitations under the License. import time +from types import ModuleType from unittest.mock import Mock, patch import arrow @@ -28,7 +29,7 @@ # For testing PrometheusPusher @pytest.fixture -def altered_metrics() -> metrics: +def altered_metrics() -> ModuleType: altered_metrics = metrics altered_metrics.delete_from_gateway = Mock() altered_metrics.pushadd_to_gateway = Mock() @@ -36,18 +37,17 @@ def altered_metrics() -> metrics: # For testing CognitePusher -class GauceSetUp: +class GaugeSetUp: gauge = Gauge("gauge", "Test gauge") @classmethod - def initGauge(self) -> None: - if GauceSetUp.gauge is None: - GauceSetUp.gauge = Gauge("gauge", "Test gauge") + def init_gauge(self) -> None: + if GaugeSetUp.gauge is None: + GaugeSetUp.gauge = Gauge("gauge", "Test gauge") -@pytest.fixture -def initGauge() -> None: - GauceSetUp.initGauge() +def init_gauge() -> None: + GaugeSetUp.init_gauge() # For testing the metrics utils @@ -55,19 +55,19 @@ def initGauge() -> None: class MyClass: - def __init__(self): + def __init__(self) -> None: global my_class_counter my_class_counter += 1 -class MyBettrerClass: - def __init__(self, value): +class AnotherClass: + def __init__(self, value: int) -> None: global my_class_counter my_class_counter += value # PrometheusPusher test -def test_normal_run(altered_metrics: metrics): +def test_normal_run(altered_metrics: ModuleType) -> None: prom = altered_metrics.PrometheusPusher(job_name="test-job", url="none", push_interval=1) last = altered_metrics.pushadd_to_gateway.call_count @@ -85,7 +85,7 @@ def test_normal_run(altered_metrics: metrics): assert altered_metrics.pushadd_to_gateway.call_count >= last -def test_error_doesnt_stop1(altered_metrics: metrics): +def test_error_doesnt_stop1(altered_metrics: ModuleType) -> None: altered_metrics.pushadd_to_gateway = Mock(side_effect=OSError) prom = altered_metrics.PrometheusPusher(job_name="test-job", url="none", push_interval=1) @@ -98,7 +98,7 @@ def test_error_doesnt_stop1(altered_metrics: metrics): prom.stop() -def test_error_doesnt_stop2(altered_metrics: metrics): +def test_error_doesnt_stop2(altered_metrics: ModuleType) -> None: altered_metrics.pushadd_to_gateway = Mock(side_effect=Exception) prom = altered_metrics.PrometheusPusher(job_name="test-job", url="none", push_interval=1) @@ -111,7 +111,7 @@ def test_error_doesnt_stop2(altered_metrics: metrics): prom.stop() -def test_clear(altered_metrics: metrics): +def test_clear(altered_metrics: ModuleType) -> None: prom = altered_metrics.PrometheusPusher(job_name="test-job", url="none", push_interval=1) prom.clear_gateway() @@ -120,8 +120,9 @@ def test_clear(altered_metrics: metrics): # CognitePusher test @patch("cognite.client.CogniteClient") -def test_init_empty_cdf(MockCogniteClient, initGauge): - client: CogniteClient = MockCogniteClient() +def test_init_empty_cdf(MockCogniteClient: Mock) -> None: + init_gauge() + client = MockCogniteClient() client.time_series.retrieve_multiple = Mock(side_effect=CogniteNotFoundError([{"externalId": "pre_gauge"}])) return_asset = Asset(id=123, external_id="asset", name="asset") @@ -147,8 +148,9 @@ def test_init_empty_cdf(MockCogniteClient, initGauge): @patch("cognite.client.CogniteClient") -def test_init_existing_asset(MockCogniteClient, initGauge): - client: CogniteClient = MockCogniteClient() +def test_init_existing_asset(MockCogniteClient: Mock) -> None: + init_gauge() + client = MockCogniteClient() client.time_series.retrieve_multiple = Mock(side_effect=CogniteNotFoundError([{"externalId": "pre_gauge"}])) return_asset = Asset(id=123, external_id="assetid", name="asset") @@ -179,8 +181,9 @@ def test_init_existing_asset(MockCogniteClient, initGauge): @patch("cognite.client.CogniteClient") -def test_init_existing_all(MockCogniteClient, initGauge): - client: CogniteClient = MockCogniteClient() +def test_init_existing_all(MockCogniteClient: Mock) -> None: + init_gauge() + client = MockCogniteClient() return_asset = Asset(id=123, external_id="assetid", name="asset") new_asset = Asset(external_id="assetid", name="asset") @@ -198,11 +201,12 @@ def test_init_existing_all(MockCogniteClient, initGauge): @patch("cognite.client.CogniteClient") -def test_push(MockCogniteClient, initGauge): +def test_push(MockCogniteClient: Mock) -> None: + init_gauge() client: CogniteClient = MockCogniteClient() pusher = CognitePusher(client, "pre_", push_interval=1) - GauceSetUp.gauge.set(5) + GaugeSetUp.gauge.set(5) pusher._push_to_server() client.time_series.data.insert_multiple.assert_called_once() @@ -219,12 +223,12 @@ def test_push(MockCogniteClient, initGauge): # MetricsUtils test @pytest.fixture -def init_counter(): +def init_counter() -> None: global my_class_counter my_class_counter = 0 -def test_safe_get(init_counter): +def test_safe_get(init_counter: None) -> None: assert my_class_counter == 0 a = safe_get(MyClass) @@ -236,6 +240,6 @@ def test_safe_get(init_counter): assert isinstance(b, MyClass) assert a == b - c = safe_get(MyBettrerClass, 5) + c = safe_get(AnotherClass, 5) assert my_class_counter == 6 - assert isinstance(c, MyBettrerClass) + assert isinstance(c, AnotherClass) diff --git a/tests/tests_unit/test_statestore.py b/tests/tests_unit/test_statestore.py index f9608f4d..b355cef7 100644 --- a/tests/tests_unit/test_statestore.py +++ b/tests/tests_unit/test_statestore.py @@ -30,7 +30,7 @@ from cognite.extractorutils.uploader import TimeSeriesUploadQueue -def test_set_state(): +def test_set_state() -> None: state_store = NoStateStore() assert "extId" not in state_store._local_state @@ -48,7 +48,7 @@ def test_set_state(): assert state_store._local_state["newExtId"] == {"low": None, "high": 7} -def test_expand_state(): +def test_expand_state() -> None: state_store = NoStateStore() assert "extId" not in state_store._local_state @@ -71,7 +71,7 @@ def test_expand_state(): assert state_store._local_state["newExtId"] == {"low": None, "high": 7} -def test_outside_state(): +def test_outside_state() -> None: state_store = NoStateStore() state_store.set_state("extId", low=3, high=10) @@ -91,7 +91,7 @@ def test_outside_state(): assert state_store.outside_state("onlyLow", 1) -def test_delete_state(): +def test_delete_state() -> None: state_store = NoStateStore() assert "extId" not in state_store._local_state @@ -104,7 +104,7 @@ def test_delete_state(): assert state_store._deleted == ["extId"] -def test_get_state(): +def test_get_state() -> None: state_store = NoStateStore() state_store._local_state = { @@ -123,7 +123,7 @@ def test_get_state(): assert state_store.get_state(["extId1", "extId3", "extId5"]) == [(1, 5), (0, None), (None, None)] -def test_local_state_interaction(): +def test_local_state_interaction() -> None: state_store = NoStateStore() state_store._local_state = { "extId1": {"low": 1, "high": 5}, @@ -173,7 +173,7 @@ def test_upload_queue_integration(MockCogniteClient: Type[CogniteClient]) -> Non @patch("cognite.client.CogniteClient") -def test_init_no_preexisting_raw(get_client_mock): +def test_init_no_preexisting_raw(get_client_mock: Mock) -> None: client: CogniteClient = get_client_mock() state_store = RawStateStore(cdf_client=client, database=DATABASE, table=TABLE) @@ -182,7 +182,7 @@ def test_init_no_preexisting_raw(get_client_mock): @patch("cognite.client.CogniteClient") -def test_init_preexisting_db(get_client_mock): +def test_init_preexisting_db(get_client_mock: Mock) -> None: client: CogniteClient = get_client_mock() client.raw.databases.create = Mock(side_effect=CogniteAPIError("", code=400)) @@ -191,7 +191,7 @@ def test_init_preexisting_db(get_client_mock): @patch("cognite.client.CogniteClient") -def test_init_preexisting_table(get_client_mock): +def test_init_preexisting_table(get_client_mock: Mock) -> None: client: CogniteClient = get_client_mock() client.raw.databases.create = Mock(side_effect=CogniteAPIError("", code=400)) client.raw.tables.create = Mock(side_effect=CogniteAPIError("", code=400)) @@ -203,7 +203,7 @@ def test_init_preexisting_table(get_client_mock): @patch("cognite.client.CogniteClient") -def test_get_raw_states_empty(get_client_mock): +def test_get_raw_states_empty(get_client_mock: Mock) -> None: client: CogniteClient = get_client_mock() state_store = RawStateStore(cdf_client=client, database=DATABASE, table=TABLE) @@ -224,7 +224,7 @@ def test_get_raw_states_empty(get_client_mock): @patch("cognite.client.CogniteClient") -def test_get_raw_states_content(get_client_mock): +def test_get_raw_states_content(get_client_mock: Mock) -> None: client: CogniteClient = get_client_mock() state_store = RawStateStore(cdf_client=client, database=DATABASE, table=TABLE) @@ -242,7 +242,7 @@ def test_get_raw_states_content(get_client_mock): @patch("cognite.client.CogniteClient") -def test_cdf_sync(get_client_mock): +def test_cdf_sync(get_client_mock: Mock) -> None: client: CogniteClient = get_client_mock() state_store = RawStateStore(cdf_client=client, database=DATABASE, table=TABLE) @@ -267,12 +267,12 @@ def test_cdf_sync(get_client_mock): assert state_store._deleted == [] -def test_init_no_file(): +def test_init_no_file() -> None: state_store = LocalStateStore("nosuchfile.json") state_store.initialize() -def test_save_and_load(): +def test_save_and_load() -> None: filename = "testfile-localstatestore.json" try: os.remove("testfile-localstatestore.json") @@ -301,7 +301,7 @@ def test_save_and_load(): os.remove(filename) -def test_start_stop(): +def test_start_stop() -> None: filename = "testfile-startstop.json" try: os.remove(filename) @@ -338,7 +338,7 @@ def test_start_stop(): os.remove(filename) -def test_invalid_file(): +def test_invalid_file() -> None: filename = "testfile-invalid.json" try: os.remove(filename) @@ -352,7 +352,7 @@ def test_invalid_file(): LocalStateStore(filename).initialize() -def test_indexing(): +def test_indexing() -> None: state_store = NoStateStore() state_store["id1"] = (1, 7) diff --git a/tests/tests_unit/test_uploader_extractor.py b/tests/tests_unit/test_uploader_extractor.py index 60b12d97..f9dca006 100644 --- a/tests/tests_unit/test_uploader_extractor.py +++ b/tests/tests_unit/test_uploader_extractor.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import datetime -from unittest.mock import patch +from unittest.mock import Mock, patch -from cognite.client import CogniteClient from cognite.client.data_classes import Row from cognite.extractorutils.uploader import EventUploadQueue, RawUploadQueue, TimeSeriesUploadQueue from cognite.extractorutils.uploader_extractor import UploaderExtractor, UploaderExtractorConfig @@ -22,8 +21,8 @@ @patch("cognite.client.CogniteClient") -def test_handle_events(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_handle_events(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() ex = UploaderExtractor[UploaderExtractorConfig]( name="ext_extractor1", description="description", config_class=UploaderExtractorConfig @@ -46,8 +45,8 @@ def test_handle_events(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_handle_raw_rows(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_handle_raw_rows(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() ex = UploaderExtractor[UploaderExtractorConfig]( name="ext_extractor2", description="description", config_class=UploaderExtractorConfig @@ -76,8 +75,8 @@ def test_handle_raw_rows(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_handle_timeseries(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_handle_timeseries(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() ex = UploaderExtractor[UploaderExtractorConfig]( name="ext_extractor3", description="description", config_class=UploaderExtractorConfig diff --git a/tests/tests_unit/test_util.py b/tests/tests_unit/test_util.py index f125d3e4..1ea4b0f5 100644 --- a/tests/tests_unit/test_util.py +++ b/tests/tests_unit/test_util.py @@ -20,7 +20,6 @@ import pytest import requests -from cognite.client import CogniteClient from cognite.client.data_classes import Asset, TimeSeries from cognite.client.exceptions import CogniteAPIError, CogniteFileUploadError, CogniteNotFoundError from cognite.extractorutils.threading import CancellationToken @@ -39,8 +38,8 @@ @patch("cognite.client.CogniteClient") -def test_ts_nothing_in_cdf(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_ts_nothing_in_cdf(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() time_series = [TimeSeries(external_id="a"), TimeSeries(external_id="b")] client.time_series.retrieve_multiple = Mock( @@ -53,8 +52,8 @@ def test_ts_nothing_in_cdf(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_ts_some_in_cdf(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_ts_some_in_cdf(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() existing = [TimeSeries(external_id="a")] new = [TimeSeries(external_id="b")] @@ -68,8 +67,8 @@ def test_ts_some_in_cdf(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_ts_all_in_cdf(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_ts_all_in_cdf(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() time_series = [TimeSeries(external_id="a"), TimeSeries(external_id="b")] ensure_time_series(client, time_series) @@ -78,8 +77,8 @@ def test_ts_all_in_cdf(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_assets_nothing_in_cdf(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_assets_nothing_in_cdf(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() assets = [Asset(external_id="a"), Asset(external_id="b")] client.assets.retrieve_multiple = Mock( @@ -92,8 +91,8 @@ def test_assets_nothing_in_cdf(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_assets_some_in_cdf(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_assets_some_in_cdf(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() existing = [Asset(external_id="a")] new = [Asset(external_id="b")] @@ -107,8 +106,8 @@ def test_assets_some_in_cdf(MockCogniteClient): @patch("cognite.client.CogniteClient") -def test_assets_all_in_cdf(MockCogniteClient): - client: CogniteClient = MockCogniteClient() +def test_assets_all_in_cdf(MockCogniteClient: Mock) -> None: + client = MockCogniteClient() assets = [Asset(external_id="a"), Asset(external_id="b")] ensure_assets(client, assets) @@ -116,14 +115,14 @@ def test_assets_all_in_cdf(MockCogniteClient): client.assets.create.assert_not_called() -def test_init(): +def test_init() -> None: with pytest.raises(TypeError): EitherId(id=123, external_id="extId") with pytest.raises(TypeError): EitherId() -def test_getters(): +def test_getters() -> None: assert EitherId(id=123).content() == 123 assert EitherId(id=123).type() == "id" assert EitherId(external_id="abc").content() == "abc" @@ -132,7 +131,7 @@ def test_getters(): assert EitherId(externalId="abc").type() == "externalId" -def test_eq(): +def test_eq() -> None: id1 = EitherId(id=123) id2 = EitherId(id=123) @@ -146,7 +145,7 @@ def test_eq(): assert id1 == id2 -def test_hash(): +def test_hash() -> None: id1 = EitherId(id=123) id2 = EitherId(id=123) @@ -158,11 +157,11 @@ def test_hash(): assert hash(id1) == hash(id2) -def test_repr(): +def test_repr() -> None: assert EitherId(externalId="extId").__repr__() == "externalId: extId" -def test_cancel(): +def test_cancel() -> None: token = CancellationToken() assert not token.is_cancelled token.cancel() @@ -172,10 +171,10 @@ def test_cancel(): assert token.wait(1) # Returns immediately. -def test_wait(): +def test_wait() -> None: token = CancellationToken() - def wait(): + def wait() -> None: token.wait() t1 = threading.Thread(target=wait) @@ -194,15 +193,15 @@ def wait(): assert not t2.is_alive() -def test_child_token(): +def test_child_token() -> None: token = CancellationToken() child_a = token.create_child_token() child_b = token.create_child_token() - def wait_a(): + def wait_a() -> None: child_a.wait() - def wait_b(): + def wait_b() -> None: child_b.wait() t1 = threading.Thread(target=wait_a) @@ -443,7 +442,7 @@ def test_timestamp_to_datetime() -> None: ("bæbæ", "bæ", 3), ], ) -def test_truncate_byte_len(test_case, expected, ln): +def test_truncate_byte_len(test_case: str, expected: str, ln: int) -> None: truncated = truncate_byte_len(test_case, ln) assert len(truncated) <= ln assert truncated == expected