Skip to content

Commit

Permalink
Fix SalesforceHook compatiblity with Pandas 2.x (apache#35145)
Browse files Browse the repository at this point in the history
In pandas 2.0.0, pandas.np was removed. Airflow 2.7.0 switched to pandas 2.x, however the SalesforceHook still has a reference to pandas.np, and is therefore broken in some cases after 2.7.0, resulting in an AttributeError if the _to_timestamp method is called.
  • Loading branch information
dave-pollock authored Oct 31, 2023
1 parent 55b015f commit 90a100a
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 3 deletions.
3 changes: 2 additions & 1 deletion airflow/providers/salesforce/hooks/salesforce.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ def _to_timestamp(cls, column: pd.Series) -> pd.Series:
# between 0 and 10 are turned into timestamps
# if the column cannot be converted,
# just return the original column untouched
import numpy as np
import pandas as pd

try:
Expand All @@ -259,7 +260,7 @@ def _to_timestamp(cls, column: pd.Series) -> pd.Series:
try:
converted.append(value.timestamp())
except (ValueError, AttributeError):
converted.append(pd.np.NaN)
converted.append(np.NaN)

return pd.Series(converted, index=column.index)

Expand Down
7 changes: 5 additions & 2 deletions tests/providers/salesforce/hooks/test_salesforce.py
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,9 @@ def test_write_object_to_file_ndjson_with_record_time(self, mock_data_frame, moc
)
@patch(
"pandas.DataFrame.from_records",
return_value=pd.DataFrame({"test": [1, 2, 3], "field_1": ["2019-01-01", "2019-01-02", "2019-01-03"]}),
return_value=pd.DataFrame(
{"test": [1, 2, 3, 4], "field_1": ["2019-01-01", "2019-01-02", "2019-01-03", "NaT"]}
),
)
def test_object_to_df_with_timestamp_conversion(self, mock_data_frame, mock_describe_object):
obj_name = "obj_name"
Expand All @@ -434,7 +436,8 @@ def test_object_to_df_with_timestamp_conversion(self, mock_data_frame, mock_desc

mock_describe_object.assert_called_once_with(obj_name)
pd.testing.assert_frame_equal(
data_frame, pd.DataFrame({"test": [1, 2, 3], "field_1": [1.546301e09, 1.546387e09, 1.546474e09]})
data_frame,
pd.DataFrame({"test": [1, 2, 3, 4], "field_1": [1.546301e09, 1.546387e09, 1.546474e09, np.nan]}),
)

@patch("airflow.providers.salesforce.hooks.salesforce.time.time", return_value=1.23)
Expand Down

0 comments on commit 90a100a

Please sign in to comment.