From dc6581209b8ac029bbd26d48e9bda8c0446b9b72 Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Fri, 29 Sep 2023 11:24:39 +0200 Subject: [PATCH 01/12] Replace row-by-row addition using .loc with a more efficient bulk addition. --- cognite/client/data_classes/_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 01bf43691c..6cd1be0509 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -164,9 +164,10 @@ def to_pandas( dumped.update(dumped.pop(key)) else: raise AssertionError(f"Could not expand attribute '{key}'") - df = pd.DataFrame(columns=["value"]) - for name, value in dumped.items(): - df.loc[name] = [value] + + data = [{"value": value} for name, value in dumped.items()] + df = pd.DataFrame(data, index=dumped.keys()) + return df def _repr_html_(self) -> str: From c30734355ccc8fe504091aad002e80b386d0bfbf Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Fri, 29 Sep 2023 13:40:12 +0200 Subject: [PATCH 02/12] Adding unit test for the new initiation of df in to_pandas() --- tests/tests_unit/test_base.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 21ec759e27..8d46c04ef3 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -370,6 +370,20 @@ def test_use_method_which_requires_cognite_client__client_not_set(self): with pytest.raises(CogniteMissingClientError): mr.use() + def test_to_pandas_method(): + import pandas as pd + + from cognite.client.data_classes import Asset + + obj = Asset(id=1) + + result_df = obj.to_pandas() + + expected_df = pd.DataFrame({"value": [1]}, index=["id"]) + + # Assert that the resultant DataFrame is equal to the expected DataFrame + pd.testing.assert_frame_equal(result_df, expected_df) + class TestCogniteFilter: def test_dump(self): From f0cc366c104ea4f090878656cf2f7a690905f796 Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Fri, 29 Sep 2023 14:05:53 +0200 Subject: [PATCH 03/12] Adding self to test_to_pandas_method --- tests/tests_unit/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 8d46c04ef3..1aff08d3a1 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -370,7 +370,7 @@ def test_use_method_which_requires_cognite_client__client_not_set(self): with pytest.raises(CogniteMissingClientError): mr.use() - def test_to_pandas_method(): + def test_to_pandas_method(self): import pandas as pd from cognite.client.data_classes import Asset From 2878ecf5b1f3e7081e4344b3f232d39556997761 Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Fri, 29 Sep 2023 14:14:08 +0200 Subject: [PATCH 04/12] Adding @pytest.mark.dsl to test_to_pandas_method --- tests/tests_unit/test_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 1aff08d3a1..8e280d59bd 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -370,6 +370,7 @@ def test_use_method_which_requires_cognite_client__client_not_set(self): with pytest.raises(CogniteMissingClientError): mr.use() + @pytest.mark.dsl def test_to_pandas_method(self): import pandas as pd From dae6fb7f7349c374b1d8725746d96fabc1a81721 Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Mon, 2 Oct 2023 09:11:06 +0200 Subject: [PATCH 05/12] Adding more elements test list and feeding dumped dict directly (to_pandas()) --- cognite/client/data_classes/_base.py | 3 +-- tests/tests_unit/test_base.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 6cd1be0509..7ae0744923 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -165,8 +165,7 @@ def to_pandas( else: raise AssertionError(f"Could not expand attribute '{key}'") - data = [{"value": value} for name, value in dumped.items()] - df = pd.DataFrame(data, index=dumped.keys()) + df = pd.DataFrame(list(dumped.values()), index=dumped.keys(), columns=["value"]) return df diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 8e280d59bd..eb1fdbb6b3 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -376,7 +376,7 @@ def test_to_pandas_method(self): from cognite.client.data_classes import Asset - obj = Asset(id=1) + obj = [Asset(id=i) for i in range(5)] result_df = obj.to_pandas() From 221e39d43ff12964e6ed989938eb46e1a720ffba Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Mon, 2 Oct 2023 11:07:22 +0200 Subject: [PATCH 06/12] Adding making the list to AssetList in test_to_pandas_method() --- tests/tests_unit/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index eb1fdbb6b3..cd243232fc 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -374,9 +374,9 @@ def test_use_method_which_requires_cognite_client__client_not_set(self): def test_to_pandas_method(self): import pandas as pd - from cognite.client.data_classes import Asset + from cognite.client.data_classes import Asset, AssetList - obj = [Asset(id=i) for i in range(5)] + obj = AssetList([Asset(id=i) for i in range(5)]) result_df = obj.to_pandas() From f9a8224cdd774e3bcd38e868b5f6a9c80c98e06c Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Mon, 2 Oct 2023 12:13:41 +0200 Subject: [PATCH 07/12] Fixing the expected df --- tests/tests_unit/test_base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index cd243232fc..5ad8e7eded 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -376,11 +376,15 @@ def test_to_pandas_method(self): from cognite.client.data_classes import Asset, AssetList - obj = AssetList([Asset(id=i) for i in range(5)]) + obj = AssetList([Asset(external_id=f"ext-{i}", name=f"name-{i}") for i in range(5)]) result_df = obj.to_pandas() - expected_df = pd.DataFrame({"value": [1]}, index=["id"]) + data = { + "external_id": ["ext-0", "ext-1", "ext-2", "ext-3", "ext-4"], + "name": ["name-0", "name-1", "name-2", "name-3", "name-4"], + } + expected_df = pd.DataFrame(data) # Assert that the resultant DataFrame is equal to the expected DataFrame pd.testing.assert_frame_equal(result_df, expected_df) From 42d42176b86694b244a72a4627a250c6077c7c1e Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Mon, 2 Oct 2023 12:23:16 +0200 Subject: [PATCH 08/12] returning to the two liner pd population --- cognite/client/data_classes/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 7ae0744923..6cd1be0509 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -165,7 +165,8 @@ def to_pandas( else: raise AssertionError(f"Could not expand attribute '{key}'") - df = pd.DataFrame(list(dumped.values()), index=dumped.keys(), columns=["value"]) + data = [{"value": value} for name, value in dumped.items()] + df = pd.DataFrame(data, index=dumped.keys()) return df From 4d320107fcbe7e0a6025be3c9b6bda17270391d1 Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Mon, 2 Oct 2023 12:29:38 +0200 Subject: [PATCH 09/12] Change DataFrame creation to use pd.DataFrame.from_dict directly --- cognite/client/data_classes/_base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 6cd1be0509..0093232747 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -165,8 +165,7 @@ def to_pandas( else: raise AssertionError(f"Could not expand attribute '{key}'") - data = [{"value": value} for name, value in dumped.items()] - df = pd.DataFrame(data, index=dumped.keys()) + df = pd.DataFrame.from_dict(dumped, orient="index", columns=["value"]) return df From 95fc54ae83f6800e51015ad0beb1c6ecddafb992 Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Mon, 2 Oct 2023 12:37:44 +0200 Subject: [PATCH 10/12] Created DataFrame from dictionary items with columns 'Name' and 'Value'. --- cognite/client/data_classes/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 0093232747..9bc5caad25 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -165,7 +165,7 @@ def to_pandas( else: raise AssertionError(f"Could not expand attribute '{key}'") - df = pd.DataFrame.from_dict(dumped, orient="index", columns=["value"]) + df = pd.DataFrame(list(dumped.items()), columns=["Name", "Value"]) return df From e8dd47989593be91fda13ea39671f8dd302061ea Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Mon, 2 Oct 2023 12:56:17 +0200 Subject: [PATCH 11/12] Created DataFrame goinf through series --- cognite/client/data_classes/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 9bc5caad25..1c9957ce28 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -165,7 +165,7 @@ def to_pandas( else: raise AssertionError(f"Could not expand attribute '{key}'") - df = pd.DataFrame(list(dumped.items()), columns=["Name", "Value"]) + df = pd.Series(dumped).to_frame(name="value") return df From d5efe279c8761e2bea75fb9ef204924a41c272aa Mon Sep 17 00:00:00 2001 From: gabor-huseb Date: Wed, 4 Oct 2023 09:05:28 +0200 Subject: [PATCH 12/12] Creating one big asset instead of list of assets. Also remove unecesary df= line before return --- cognite/client/data_classes/_base.py | 4 +--- tests/tests_unit/test_base.py | 30 +++++++++++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 1c9957ce28..9202c31b5c 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -165,9 +165,7 @@ def to_pandas( else: raise AssertionError(f"Could not expand attribute '{key}'") - df = pd.Series(dumped).to_frame(name="value") - - return df + return pd.Series(dumped).to_frame(name="value") def _repr_html_(self) -> str: return notebook_display_with_fallback(self) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 5ad8e7eded..3832d8d526 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -374,17 +374,33 @@ def test_use_method_which_requires_cognite_client__client_not_set(self): def test_to_pandas_method(self): import pandas as pd - from cognite.client.data_classes import Asset, AssetList - - obj = AssetList([Asset(external_id=f"ext-{i}", name=f"name-{i}") for i in range(5)]) + from cognite.client.data_classes import Asset, Label + + asset = Asset( + external_id="test-1", + name="test 1", + parent_external_id="parent-test-1", + description="A test asset", + data_set_id=123, + labels=[Label(external_id="ROTATING_EQUIPMENT", name="Rotating equipment")], + ) - result_df = obj.to_pandas() + result_df = asset.to_pandas() data = { - "external_id": ["ext-0", "ext-1", "ext-2", "ext-3", "ext-4"], - "name": ["name-0", "name-1", "name-2", "name-3", "name-4"], + "value": [ + "test-1", + "test 1", + "parent-test-1", + "A test asset", + 123, + [{"externalId": "ROTATING_EQUIPMENT", "name": "Rotating equipment"}], + ] } - expected_df = pd.DataFrame(data) + + index_labels = ["external_id", "name", "parent_external_id", "description", "data_set_id", "labels"] + + expected_df = pd.DataFrame(data, index=index_labels) # Assert that the resultant DataFrame is equal to the expected DataFrame pd.testing.assert_frame_equal(result_df, expected_df)