From e36bf8b8f3345c22c7534514e390093c5511f95b Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Fri, 9 Feb 2024 19:53:04 +0200 Subject: [PATCH 1/7] Add logs for the failed check command --- .../source_amazon_seller_partner/source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index b6547b208146..de4dee574db4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -3,6 +3,7 @@ # +import traceback from os import getenv from typing import Any, List, Mapping, Optional, Tuple From e445992bd3f92ed6bfc6816d2b5aed0ce58db879 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Fri, 9 Feb 2024 19:55:58 +0200 Subject: [PATCH 2/7] Update version --- .../metadata.yaml | 2 +- .../sources/amazon-seller-partner.md | 137 +++++++++--------- 2 files changed, 70 insertions(+), 69 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index c1232b46e330..ffbc4a51e268 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -15,7 +15,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 3.4.0 + dockerImageTag: 3.5.0 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 1ceaa780c971..e5b1bfb164ba 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -164,71 +164,72 @@ Information about rate limits you may find [here](https://developer-docs.amazon. ## Changelog -| Version | Date | Pull Request | Subject | -|:---------|:-----------|:------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `3.4.0` | 2024-02-15 | [\#35273](https://github.com/airbytehq/airbyte/pull/35273) | Add `VendorOrders` stream | -| `3.3.2` | 2024-02-13 | [\#33996](https://github.com/airbytehq/airbyte/pull/33996) | Add integration tests | -| `3.3.1` | 2024-02-09 | [\#35106](https://github.com/airbytehq/airbyte/pull/35106) | Add logs for the failed check command | -| `3.3.0` | 2024-02-09 | [\#35062](https://github.com/airbytehq/airbyte/pull/35062) | Fix the check command for the `Vendor` account type | -| `3.2.2` | 2024-02-07 | [\#34914](https://github.com/airbytehq/airbyte/pull/34914) | Fix date formatting for ledger reports with aggregation by month | -| `3.2.1` | 2024-01-30 | [\#34654](https://github.com/airbytehq/airbyte/pull/34654) | Fix date format in state message for streams with custom dates formatting | -| `3.2.0` | 2024-01-26 | [\#34549](https://github.com/airbytehq/airbyte/pull/34549) | Update schemas for vendor analytics streams | -| `3.1.0` | 2024-01-17 | [\#34283](https://github.com/airbytehq/airbyte/pull/34283) | Delete deprecated streams | -| `3.0.1` | 2023-12-22 | [\#33741](https://github.com/airbytehq/airbyte/pull/33741) | Improve report streams performance | -| `3.0.0` | 2023-12-12 | [\#32977](https://github.com/airbytehq/airbyte/pull/32977) | Make all streams incremental | -| `2.5.0` | 2023-11-27 | [\#32505](https://github.com/airbytehq/airbyte/pull/32505) | Make report options configurable via UI | -| `2.4.0` | 2023-11-23 | [\#32738](https://github.com/airbytehq/airbyte/pull/32738) | Add `GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT`, `GET_VENDOR_REAL_TIME_INVENTORY_REPORT`, and `GET_VENDOR_TRAFFIC_REPORT` streams | -| `2.3.0` | 2023-11-22 | [\#32541](https://github.com/airbytehq/airbyte/pull/32541) | Make `GET_AFN_INVENTORY_DATA`, `GET_AFN_INVENTORY_DATA_BY_COUNTRY`, and `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` streams incremental | -| `2.2.0` | 2023-11-21 | [\#32639](https://github.com/airbytehq/airbyte/pull/32639) | Make start date optional, if start date is not provided, date 2 years ago from today will be used | -| `2.1.1` | 2023-11-21 | [\#32560](https://github.com/airbytehq/airbyte/pull/32560) | Silently exit sync if the retry attempts were unsuccessful | -| `2.1.0` | 2023-11-21 | [\#32591](https://github.com/airbytehq/airbyte/pull/32591) | Add new fields to GET_LEDGER_DETAIL_VIEW_DATA, GET_FBA_INVENTORY_PLANNING_DATA and Orders schemas | -| `2.0.2` | 2023-11-17 | [\#32462](https://github.com/airbytehq/airbyte/pull/32462) | Remove Max time option from specification; set default waiting time for reports to 1 hour | -| `2.0.1` | 2023-11-16 | [\#32550](https://github.com/airbytehq/airbyte/pull/32550) | Fix the OAuth flow | -| `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | -| `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | -| `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | -| `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | -| `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | -| `1.5.0` | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054) | Add new stream `OrderItems` | -| `1.4.1` | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050) | Fix - non vendor accounts connector create/check issue | -| `1.4.0` | 2023-07-21 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Add `GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING` and `GET_ORDER_REPORT_DATA_SHIPPING` streams | -| `1.3.0` | 2023-06-09 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Removed `app_id` from `InputConfiguration`, refactored `spec` | -| `1.2.0` | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration | -| `1.1.0` | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream | -| `1.0.1` | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace | -| `1.0.0` | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts | -| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606) | Implement reportOptions for all missing reports and refactor | -| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300) | Make AWS Access Key, AWS Secret Access and Role ARN optional | -| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430) | Implement slicing for report streams | -| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896) | Validate connections without orders data | -| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581) | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT | -| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283) | Added multiple (22) report types | -| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629) | Report API version to 2021-06-30, added multiple (5) report types | -| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063) | Add Restock Inventory Report | -| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625) | Add FBA Storage Fees Report | -| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604) | Add new streams: Fullfiments returns and Settlement reports | -| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633) | Fix - handle start date for financial stream | -| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364) | Add financial streams | -| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059) | Add replication end date to config | -| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support | -| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report | -| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams | -| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role | -| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports | -| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report | -| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime | -| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams | -| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders | -| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning | -| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names | -| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format | -| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter | -| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report | -| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | -| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability | -| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams | -| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders | -| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | Added `extra stream` support. Updated `reports streams` logics | -| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | Rebuild source with `airbyte-cdk` | -| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | Bugfix failing `connection check` | -| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | Fixed: Sync fails with timeout when create report is CANCELLED` | +| Version | Date | Pull Request | Subject | +|:---------|:-----------|:------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `3.5.0` | 2024-02-09 | [\#35106](https://github.com/airbytehq/airbyte/pull/35106) | Add logs for the failed check command | +| `3.4.0` | 2024-02-15 | [\#35273](https://github.com/airbytehq/airbyte/pull/35273) | Add `VendorOrders` stream | +| `3.3.2` | 2024-02-13 | [\#33996](https://github.com/airbytehq/airbyte/pull/33996) | Add integration tests | +| `3.3.1` | 2024-02-09 | [\#35106](https://github.com/airbytehq/airbyte/pull/35106) | Add logs for the failed check command | +| `3.3.0` | 2024-02-09 | [\#35062](https://github.com/airbytehq/airbyte/pull/35062) | Fix the check command for the `Vendor` account type | +| `3.2.2` | 2024-02-07 | [\#34914](https://github.com/airbytehq/airbyte/pull/34914) | Fix date formatting for ledger reports with aggregation by month | +| `3.2.1` | 2024-01-30 | [\#34654](https://github.com/airbytehq/airbyte/pull/34654) | Fix date format in state message for streams with custom dates formatting | +| `3.2.0` | 2024-01-26 | [\#34549](https://github.com/airbytehq/airbyte/pull/34549) | Update schemas for vendor analytics streams | +| `3.1.0` | 2024-01-17 | [\#34283](https://github.com/airbytehq/airbyte/pull/34283) | Delete deprecated streams | +| `3.0.1` | 2023-12-22 | [\#33741](https://github.com/airbytehq/airbyte/pull/33741) | Improve report streams performance | +| `3.0.0` | 2023-12-12 | [\#32977](https://github.com/airbytehq/airbyte/pull/32977) | Make all streams incremental | +| `2.5.0` | 2023-11-27 | [\#32505](https://github.com/airbytehq/airbyte/pull/32505) | Make report options configurable via UI | +| `2.4.0` | 2023-11-23 | [\#32738](https://github.com/airbytehq/airbyte/pull/32738) | Add `GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT`, `GET_VENDOR_REAL_TIME_INVENTORY_REPORT`, and `GET_VENDOR_TRAFFIC_REPORT` streams | +| `2.3.0` | 2023-11-22 | [\#32541](https://github.com/airbytehq/airbyte/pull/32541) | Make `GET_AFN_INVENTORY_DATA`, `GET_AFN_INVENTORY_DATA_BY_COUNTRY`, and `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` streams incremental | +| `2.2.0` | 2023-11-21 | [\#32639](https://github.com/airbytehq/airbyte/pull/32639) | Make start date optional, if start date is not provided, date 2 years ago from today will be used | +| `2.1.1` | 2023-11-21 | [\#32560](https://github.com/airbytehq/airbyte/pull/32560) | Silently exit sync if the retry attempts were unsuccessful | +| `2.1.0` | 2023-11-21 | [\#32591](https://github.com/airbytehq/airbyte/pull/32591) | Add new fields to GET_LEDGER_DETAIL_VIEW_DATA, GET_FBA_INVENTORY_PLANNING_DATA and Orders schemas | +| `2.0.2` | 2023-11-17 | [\#32462](https://github.com/airbytehq/airbyte/pull/32462) | Remove Max time option from specification; set default waiting time for reports to 1 hour | +| `2.0.1` | 2023-11-16 | [\#32550](https://github.com/airbytehq/airbyte/pull/32550) | Fix the OAuth flow | +| `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | +| `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | +| `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | +| `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | +| `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | +| `1.5.0` | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054) | Add new stream `OrderItems` | +| `1.4.1` | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050) | Fix - non vendor accounts connector create/check issue | +| `1.4.0` | 2023-07-21 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Add `GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING` and `GET_ORDER_REPORT_DATA_SHIPPING` streams | +| `1.3.0` | 2023-06-09 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Removed `app_id` from `InputConfiguration`, refactored `spec` | +| `1.2.0` | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration | +| `1.1.0` | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream | +| `1.0.1` | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace | +| `1.0.0` | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts | +| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606) | Implement reportOptions for all missing reports and refactor | +| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300) | Make AWS Access Key, AWS Secret Access and Role ARN optional | +| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430) | Implement slicing for report streams | +| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896) | Validate connections without orders data | +| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581) | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT | +| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283) | Added multiple (22) report types | +| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629) | Report API version to 2021-06-30, added multiple (5) report types | +| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063) | Add Restock Inventory Report | +| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625) | Add FBA Storage Fees Report | +| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604) | Add new streams: Fullfiments returns and Settlement reports | +| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633) | Fix - handle start date for financial stream | +| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364) | Add financial streams | +| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059) | Add replication end date to config | +| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support | +| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report | +| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams | +| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role | +| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports | +| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report | +| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime | +| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams | +| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders | +| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning | +| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names | +| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format | +| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter | +| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report | +| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | +| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability | +| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams | +| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders | +| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | Added `extra stream` support. Updated `reports streams` logics | +| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | Rebuild source with `airbyte-cdk` | +| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | Bugfix failing `connection check` | +| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | Fixed: Sync fails with timeout when create report is CANCELLED` | From a55275aa0ba442fb300039d6c1150811aa0fbe34 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Mon, 12 Feb 2024 19:33:55 +0200 Subject: [PATCH 3/7] Add error description to the failed reports --- .../source_amazon_seller_partner/source.py | 5 +++-- .../source_amazon_seller_partner/streams.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index de4dee574db4..403b5a4d6f47 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -116,10 +116,11 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> if config.get("account_type", "Seller") == "Seller": stream_to_check = Orders(**stream_kwargs) + next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh)) else: stream_to_check = VendorSalesReports(**stream_kwargs) - - next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh)) + stream_slices = list(stream_to_check.stream_slices(sync_mode=SyncMode.full_refresh)) + next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices[0])) return True, None except Exception as e: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index a8a644709ffa..845e58208191 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -393,6 +393,18 @@ def read_records( record["dataEndTime"] = report_end_date.strftime(DATE_FORMAT) yield record elif processing_status == ReportProcessingStatus.FATAL: + # retrieve and decrypt the report document + document_id = report_payload["reportDocumentId"] + request_headers = self.request_headers() + request = self._create_prepared_request( + path=self.path(document_id=document_id), + headers=dict(request_headers, **self.authenticator.get_auth_header()), + params=self.request_params(), + ) + response = self._send_request(request, {}) + + document = self.download_and_decompress_report_document(response.json()) + print(document) raise AirbyteTracedException( internal_message=( f"Failed to retrieve the report '{self.name}' for period " From 0ef3cef90e1c64d63f0b69b888abd22d0cdbb9b1 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 15 Feb 2024 21:18:37 +0200 Subject: [PATCH 4/7] Fix Failed report handling --- .../source_amazon_seller_partner/source.py | 2 +- .../source_amazon_seller_partner/streams.py | 42 ++++++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index 403b5a4d6f47..91c2e1bd80ec 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -118,7 +118,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> stream_to_check = Orders(**stream_kwargs) next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh)) else: - stream_to_check = VendorSalesReports(**stream_kwargs) + stream_to_check = VendorOrders(**stream_kwargs) stream_slices = list(stream_to_check.stream_slices(sync_mode=SyncMode.full_refresh)) next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices[0])) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 845e58208191..83cd4b4663f4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -6,6 +6,7 @@ import csv import gzip import json +import logging import os import time from abc import ABC, abstractmethod @@ -282,6 +283,15 @@ def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: return report_payload + def _retrieve_report_result(self, report_document_id: str) -> requests.Response: + request_headers = self.request_headers() + request = self._create_prepared_request( + path=self.path(document_id=report_document_id), + headers=dict(request_headers, **self.authenticator.get_auth_header()), + params=self.request_params(), + ) + return self._send_request(request, {}) + @default_backoff_handler(factor=5, max_tries=5) def download_and_decompress_report_document(self, payload: dict) -> str: """ @@ -381,35 +391,29 @@ def read_records( if processing_status == ReportProcessingStatus.DONE: # retrieve and decrypt the report document document_id = report_payload["reportDocumentId"] - request_headers = self.request_headers() - request = self._create_prepared_request( - path=self.path(document_id=document_id), - headers=dict(request_headers, **self.authenticator.get_auth_header()), - params=self.request_params(), - ) - response = self._send_request(request, {}) + response = self._retrieve_report_result(document_id) + for record in self.parse_response(response, stream_state, stream_slice): if report_end_date: record["dataEndTime"] = report_end_date.strftime(DATE_FORMAT) yield record elif processing_status == ReportProcessingStatus.FATAL: # retrieve and decrypt the report document - document_id = report_payload["reportDocumentId"] - request_headers = self.request_headers() - request = self._create_prepared_request( - path=self.path(document_id=document_id), - headers=dict(request_headers, **self.authenticator.get_auth_header()), - params=self.request_params(), - ) - response = self._send_request(request, {}) + try: + document_id = report_payload["reportDocumentId"] + response = self._retrieve_report_result(document_id) + + document = self.download_and_decompress_report_document(response.json()) + error_response = json.loads(document) + except Exception as e: + logging.error(f"Failed to retrieve the report result document for stream '{self.name}'. Exception: {e}") + error_response = "Failed to retrieve the report result document." - document = self.download_and_decompress_report_document(response.json()) - print(document) raise AirbyteTracedException( internal_message=( f"Failed to retrieve the report '{self.name}' for period " - f"{stream_slice['dataStartTime']}-{stream_slice['dataEndTime']} " - "due to Amazon Seller Partner platform issues. This will be read during the next sync." + f"{stream_slice['dataStartTime']}-{stream_slice['dataEndTime']}. " + f"This will be read during the next sync. Error: {error_response}" ) ) elif processing_status == ReportProcessingStatus.CANCELLED: From defd95b8016289a2c81d03cd753478332afe443e Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Thu, 15 Feb 2024 21:27:30 +0200 Subject: [PATCH 5/7] Update version --- .../sources/amazon-seller-partner.md | 138 +++++++++--------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index e5b1bfb164ba..5933cb0d5fb3 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -164,72 +164,72 @@ Information about rate limits you may find [here](https://developer-docs.amazon. ## Changelog -| Version | Date | Pull Request | Subject | -|:---------|:-----------|:------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `3.5.0` | 2024-02-09 | [\#35106](https://github.com/airbytehq/airbyte/pull/35106) | Add logs for the failed check command | -| `3.4.0` | 2024-02-15 | [\#35273](https://github.com/airbytehq/airbyte/pull/35273) | Add `VendorOrders` stream | -| `3.3.2` | 2024-02-13 | [\#33996](https://github.com/airbytehq/airbyte/pull/33996) | Add integration tests | -| `3.3.1` | 2024-02-09 | [\#35106](https://github.com/airbytehq/airbyte/pull/35106) | Add logs for the failed check command | -| `3.3.0` | 2024-02-09 | [\#35062](https://github.com/airbytehq/airbyte/pull/35062) | Fix the check command for the `Vendor` account type | -| `3.2.2` | 2024-02-07 | [\#34914](https://github.com/airbytehq/airbyte/pull/34914) | Fix date formatting for ledger reports with aggregation by month | -| `3.2.1` | 2024-01-30 | [\#34654](https://github.com/airbytehq/airbyte/pull/34654) | Fix date format in state message for streams with custom dates formatting | -| `3.2.0` | 2024-01-26 | [\#34549](https://github.com/airbytehq/airbyte/pull/34549) | Update schemas for vendor analytics streams | -| `3.1.0` | 2024-01-17 | [\#34283](https://github.com/airbytehq/airbyte/pull/34283) | Delete deprecated streams | -| `3.0.1` | 2023-12-22 | [\#33741](https://github.com/airbytehq/airbyte/pull/33741) | Improve report streams performance | -| `3.0.0` | 2023-12-12 | [\#32977](https://github.com/airbytehq/airbyte/pull/32977) | Make all streams incremental | -| `2.5.0` | 2023-11-27 | [\#32505](https://github.com/airbytehq/airbyte/pull/32505) | Make report options configurable via UI | -| `2.4.0` | 2023-11-23 | [\#32738](https://github.com/airbytehq/airbyte/pull/32738) | Add `GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT`, `GET_VENDOR_REAL_TIME_INVENTORY_REPORT`, and `GET_VENDOR_TRAFFIC_REPORT` streams | -| `2.3.0` | 2023-11-22 | [\#32541](https://github.com/airbytehq/airbyte/pull/32541) | Make `GET_AFN_INVENTORY_DATA`, `GET_AFN_INVENTORY_DATA_BY_COUNTRY`, and `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` streams incremental | -| `2.2.0` | 2023-11-21 | [\#32639](https://github.com/airbytehq/airbyte/pull/32639) | Make start date optional, if start date is not provided, date 2 years ago from today will be used | -| `2.1.1` | 2023-11-21 | [\#32560](https://github.com/airbytehq/airbyte/pull/32560) | Silently exit sync if the retry attempts were unsuccessful | -| `2.1.0` | 2023-11-21 | [\#32591](https://github.com/airbytehq/airbyte/pull/32591) | Add new fields to GET_LEDGER_DETAIL_VIEW_DATA, GET_FBA_INVENTORY_PLANNING_DATA and Orders schemas | -| `2.0.2` | 2023-11-17 | [\#32462](https://github.com/airbytehq/airbyte/pull/32462) | Remove Max time option from specification; set default waiting time for reports to 1 hour | -| `2.0.1` | 2023-11-16 | [\#32550](https://github.com/airbytehq/airbyte/pull/32550) | Fix the OAuth flow | -| `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | -| `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | -| `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | -| `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | -| `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | -| `1.5.0` | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054) | Add new stream `OrderItems` | -| `1.4.1` | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050) | Fix - non vendor accounts connector create/check issue | -| `1.4.0` | 2023-07-21 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Add `GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING` and `GET_ORDER_REPORT_DATA_SHIPPING` streams | -| `1.3.0` | 2023-06-09 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Removed `app_id` from `InputConfiguration`, refactored `spec` | -| `1.2.0` | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration | -| `1.1.0` | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream | -| `1.0.1` | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace | -| `1.0.0` | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts | -| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606) | Implement reportOptions for all missing reports and refactor | -| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300) | Make AWS Access Key, AWS Secret Access and Role ARN optional | -| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430) | Implement slicing for report streams | -| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896) | Validate connections without orders data | -| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581) | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT | -| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283) | Added multiple (22) report types | -| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629) | Report API version to 2021-06-30, added multiple (5) report types | -| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063) | Add Restock Inventory Report | -| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625) | Add FBA Storage Fees Report | -| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604) | Add new streams: Fullfiments returns and Settlement reports | -| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633) | Fix - handle start date for financial stream | -| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364) | Add financial streams | -| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059) | Add replication end date to config | -| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support | -| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report | -| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams | -| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role | -| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports | -| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report | -| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime | -| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams | -| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders | -| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning | -| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names | -| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format | -| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter | -| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report | -| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | -| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability | -| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams | -| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders | -| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | Added `extra stream` support. Updated `reports streams` logics | -| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | Rebuild source with `airbyte-cdk` | -| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | Bugfix failing `connection check` | -| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | Fixed: Sync fails with timeout when create report is CANCELLED` | +| Version | Date | Pull Request | Subject | +|:---------|:-----------|:------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `3.5.0` | 2024-02-09 | [\#35331](https://github.com/airbytehq/airbyte/pull/35331) | Fix check for Vendor accounts. Add failed report result message | +| `3.4.0` | 2024-02-15 | [\#35273](https://github.com/airbytehq/airbyte/pull/35273) | Add `VendorOrders` stream | +| `3.3.2` | 2024-02-13 | [\#33996](https://github.com/airbytehq/airbyte/pull/33996) | Add integration tests | +| `3.3.1` | 2024-02-09 | [\#35106](https://github.com/airbytehq/airbyte/pull/35106) | Add logs for the failed check command | +| `3.3.0` | 2024-02-09 | [\#35062](https://github.com/airbytehq/airbyte/pull/35062) | Fix the check command for the `Vendor` account type | +| `3.2.2` | 2024-02-07 | [\#34914](https://github.com/airbytehq/airbyte/pull/34914) | Fix date formatting for ledger reports with aggregation by month | +| `3.2.1` | 2024-01-30 | [\#34654](https://github.com/airbytehq/airbyte/pull/34654) | Fix date format in state message for streams with custom dates formatting | +| `3.2.0` | 2024-01-26 | [\#34549](https://github.com/airbytehq/airbyte/pull/34549) | Update schemas for vendor analytics streams | +| `3.1.0` | 2024-01-17 | [\#34283](https://github.com/airbytehq/airbyte/pull/34283) | Delete deprecated streams | +| `3.0.1` | 2023-12-22 | [\#33741](https://github.com/airbytehq/airbyte/pull/33741) | Improve report streams performance | +| `3.0.0` | 2023-12-12 | [\#32977](https://github.com/airbytehq/airbyte/pull/32977) | Make all streams incremental | +| `2.5.0` | 2023-11-27 | [\#32505](https://github.com/airbytehq/airbyte/pull/32505) | Make report options configurable via UI | +| `2.4.0` | 2023-11-23 | [\#32738](https://github.com/airbytehq/airbyte/pull/32738) | Add `GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT`, `GET_VENDOR_REAL_TIME_INVENTORY_REPORT`, and `GET_VENDOR_TRAFFIC_REPORT` streams | +| `2.3.0` | 2023-11-22 | [\#32541](https://github.com/airbytehq/airbyte/pull/32541) | Make `GET_AFN_INVENTORY_DATA`, `GET_AFN_INVENTORY_DATA_BY_COUNTRY`, and `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` streams incremental | +| `2.2.0` | 2023-11-21 | [\#32639](https://github.com/airbytehq/airbyte/pull/32639) | Make start date optional, if start date is not provided, date 2 years ago from today will be used | +| `2.1.1` | 2023-11-21 | [\#32560](https://github.com/airbytehq/airbyte/pull/32560) | Silently exit sync if the retry attempts were unsuccessful | +| `2.1.0` | 2023-11-21 | [\#32591](https://github.com/airbytehq/airbyte/pull/32591) | Add new fields to GET_LEDGER_DETAIL_VIEW_DATA, GET_FBA_INVENTORY_PLANNING_DATA and Orders schemas | +| `2.0.2` | 2023-11-17 | [\#32462](https://github.com/airbytehq/airbyte/pull/32462) | Remove Max time option from specification; set default waiting time for reports to 1 hour | +| `2.0.1` | 2023-11-16 | [\#32550](https://github.com/airbytehq/airbyte/pull/32550) | Fix the OAuth flow | +| `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | +| `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | +| `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | +| `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | +| `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | +| `1.5.0` | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054) | Add new stream `OrderItems` | +| `1.4.1` | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050) | Fix - non vendor accounts connector create/check issue | +| `1.4.0` | 2023-07-21 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Add `GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING` and `GET_ORDER_REPORT_DATA_SHIPPING` streams | +| `1.3.0` | 2023-06-09 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Removed `app_id` from `InputConfiguration`, refactored `spec` | +| `1.2.0` | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration | +| `1.1.0` | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream | +| `1.0.1` | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace | +| `1.0.0` | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts | +| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606) | Implement reportOptions for all missing reports and refactor | +| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300) | Make AWS Access Key, AWS Secret Access and Role ARN optional | +| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430) | Implement slicing for report streams | +| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896) | Validate connections without orders data | +| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581) | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT | +| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283) | Added multiple (22) report types | +| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629) | Report API version to 2021-06-30, added multiple (5) report types | +| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063) | Add Restock Inventory Report | +| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625) | Add FBA Storage Fees Report | +| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604) | Add new streams: Fullfiments returns and Settlement reports | +| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633) | Fix - handle start date for financial stream | +| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364) | Add financial streams | +| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059) | Add replication end date to config | +| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support | +| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report | +| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams | +| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role | +| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports | +| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report | +| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime | +| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams | +| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders | +| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning | +| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names | +| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format | +| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter | +| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report | +| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | +| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability | +| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams | +| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders | +| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | Added `extra stream` support. Updated `reports streams` logics | +| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | Rebuild source with `airbyte-cdk` | +| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | Bugfix failing `connection check` | +| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | Fixed: Sync fails with timeout when create report is CANCELLED` | From 97f48b572041c79325f8d1d99072256bd8cf726c Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Mon, 19 Feb 2024 14:44:22 +0200 Subject: [PATCH 6/7] Update docs and fix tests --- .../unit_tests/integration/request_builder.py | 8 +-- .../integration/test_report_based_streams.py | 57 +++++++++++-------- ...test_vendor_direct_fulfillment_shipping.py | 48 +++++++--------- .../integration/test_vendor_orders.py | 28 +++------ .../unit_tests/integration/utils.py | 8 +-- .../unit_tests/test_migrations.py | 1 + .../unit_tests/test_source.py | 4 +- .../unit_tests/test_streams.py | 17 +++--- .../sources/amazon-seller-partner.md | 2 + 9 files changed, 76 insertions(+), 97 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/request_builder.py index 81f4f922f8f2..d14097dc3cd1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/request_builder.py @@ -14,17 +14,13 @@ class RequestBuilder: - @classmethod def auth_endpoint(cls) -> RequestBuilder: request_headers = {"Content-Type": "application/x-www-form-urlencoded"} request_body = ( - f"grant_type=refresh_token&client_id={LWA_APP_ID}&" - f"client_secret={LWA_CLIENT_SECRET}&refresh_token={REFRESH_TOKEN}" - ) - return cls("auth/o2/token").with_base_url("https://api.amazon.com").with_headers(request_headers).with_body( - request_body + f"grant_type=refresh_token&client_id={LWA_APP_ID}&" f"client_secret={LWA_CLIENT_SECRET}&refresh_token={REFRESH_TOKEN}" ) + return cls("auth/o2/token").with_base_url("https://api.amazon.com").with_headers(request_headers).with_body(request_body) @classmethod def create_report_endpoint(cls, report_name: str) -> RequestBuilder: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py index 796d699d33d2..5c71bd43b428 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py @@ -118,11 +118,11 @@ def _check_report_status_response( "dataEndTime": CONFIG_END_DATE, "createdTime": CONFIG_START_DATE, "dataStartTime": CONFIG_START_DATE, + "reportDocumentId": report_document_id, } if processing_status == ReportProcessingStatus.DONE: response_body.update( { - "reportDocumentId": report_document_id, "processingEndTime": CONFIG_START_DATE, "processingStartTime": CONFIG_START_DATE, } @@ -141,18 +141,22 @@ def _get_document_download_url_response( return build_response(response_body, status_code=HTTPStatus.OK) -def _download_document_response( - stream_name: str, data_format: Optional[str] = "csv", compressed: Optional[bool] = False -) -> HttpResponse: +def _download_document_response(stream_name: str, data_format: Optional[str] = "csv", compressed: Optional[bool] = False) -> HttpResponse: response_body = find_template(stream_name, __file__, data_format) if compressed: response_body = gzip.compress(response_body.encode("iso-8859-1")) return HttpResponse(body=response_body, status_code=HTTPStatus.OK) +def _download_document_error_response(compressed: Optional[bool] = False) -> HttpResponse: + response_body = '{"errorDetails":"Error in report request: This report type requires the reportPeriod, distributorView, sellingProgram reportOption to be specified. Please review the document for this report type on GitHub, provide a value for this reportOption in your request, and try again."}' + if compressed: + response_body = gzip.compress(response_body.encode("iso-8859-1")) + return HttpResponse(body=response_body, status_code=HTTPStatus.OK) + + @freezegun.freeze_time(NOW.isoformat()) class TestFullRefresh: - @staticmethod def _read(stream_name: str, config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -164,9 +168,7 @@ def _read(stream_name: str, config_: ConfigBuilder, expecting_exception: bool = @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) @HttpMocker() - def test_given_report_when_read_then_return_records( - self, stream_name: str, data_format: str, http_mocker: HttpMocker - ) -> None: + def test_given_report_when_read_then_return_records(self, stream_name: str, data_format: str, http_mocker: HttpMocker) -> None: mock_auth(http_mocker) http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) @@ -329,9 +331,7 @@ def test_given_report_access_forbidden_when_read_then_no_records_and_error_logge ) -> None: mock_auth(http_mocker) - http_mocker.post( - _create_report_request(stream_name).build(), response_with_status(status_code=HTTPStatus.FORBIDDEN) - ) + http_mocker.post(_create_report_request(stream_name).build(), response_with_status(status_code=HTTPStatus.FORBIDDEN)) output = self._read(stream_name, config()) message_on_access_forbidden = ( @@ -354,9 +354,7 @@ def test_given_report_status_cancelled_when_read_then_stream_completed_successfu _check_report_status_response(stream_name, processing_status=ReportProcessingStatus.CANCELLED), ) - message_on_report_cancelled = ( - f"The report for stream '{stream_name}' was cancelled or there is no data to return." - ) + message_on_report_cancelled = f"The report for stream '{stream_name}' was cancelled or there is no data to return." output = self._read(stream_name, config()) assert_message_in_log_output(message_on_report_cancelled, output) @@ -372,15 +370,28 @@ def test_given_report_status_fatal_when_read_then_exception_raised( http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) http_mocker.get( _check_report_status_request(_REPORT_ID).build(), - _check_report_status_response(stream_name, processing_status=ReportProcessingStatus.FATAL), + _check_report_status_response( + stream_name, processing_status=ReportProcessingStatus.FATAL, report_document_id=_REPORT_DOCUMENT_ID + ), + ) + + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + [ + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + _download_document_error_response(), + ], ) output = self._read(stream_name, config(), expecting_exception=True) - assert output.errors[-1].trace.error.failure_type == FailureType.config_error + assert output.errors[-1].trace.error.failure_type == FailureType.system_error assert ( - f"Failed to retrieve the report '{stream_name}' for period {CONFIG_START_DATE}-{CONFIG_END_DATE} " - "due to Amazon Seller Partner platform issues. This will be read during the next sync." - ) in output.errors[-1].trace.error.message + f"Failed to retrieve the report '{stream_name}' for period {CONFIG_START_DATE}-{CONFIG_END_DATE}. This will be read during the next sync. Error: {{'errorDetails': 'Error in report request: This report type requires the reportPeriod, distributorView, sellingProgram reportOption to be specified. Please review the document for this report type on GitHub, provide a value for this reportOption in your request, and try again.'}}" + ) in output.errors[-1].trace.error.internal_message @pytest.mark.parametrize( ("stream_name", "date_field", "expected_date_value"), @@ -405,9 +416,7 @@ def test_given_report_with_incorrect_date_format_when_read_then_formatted( _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), ) - http_mocker.get( - _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), _download_document_response(stream_name) - ) + http_mocker.get(_download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), _download_document_response(stream_name)) output = self._read(stream_name, config()) assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS @@ -425,9 +434,7 @@ def test_given_http_error_500_on_create_report_when_read_then_no_records_and_err response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), ) - message_on_backoff_exception = ( - f"The report for stream '{stream_name}' was cancelled due to several failed retry attempts." - ) + message_on_backoff_exception = f"The report for stream '{stream_name}' was cancelled due to several failed retry attempts." output = self._read(stream_name, config()) assert_message_in_log_output(message_on_backoff_exception, output) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py index f210a920bfeb..c9755255b550 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py @@ -63,7 +63,6 @@ def _shipping_label_record() -> RecordBuilder: @freezegun.freeze_time(NOW.isoformat()) class TestFullRefresh: - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -89,9 +88,7 @@ def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMo mock_auth(http_mocker) http_mocker.get( _vendor_direct_fulfillment_shipping_request().build(), - _vendor_direct_fulfillment_shipping_response().with_pagination().with_record( - _shipping_label_record() - ).build(), + _vendor_direct_fulfillment_shipping_response().with_pagination().with_record(_shipping_label_record()).build(), ) query_params_with_next_page_token = { _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), @@ -100,9 +97,10 @@ def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMo } http_mocker.get( _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_with_next_page_token).build(), - _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).with_record( - _shipping_label_record() - ).build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), ) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) @@ -135,9 +133,7 @@ def test_given_two_slices_when_read_then_return_records(self, http_mocker: HttpM assert len(output.records) == 2 @HttpMocker() - def test_given_http_status_500_then_200_when_read_then_retry_and_return_records( - self, http_mocker: HttpMocker - ) -> None: + def test_given_http_status_500_then_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: mock_auth(http_mocker) http_mocker.get( _vendor_direct_fulfillment_shipping_request().build(), @@ -151,9 +147,7 @@ def test_given_http_status_500_then_200_when_read_then_retry_and_return_records( assert len(output.records) == 1 @HttpMocker() - def test_given_http_status_500_on_availability_when_read_then_raise_system_error( - self, http_mocker: HttpMocker - ) -> None: + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: mock_auth(http_mocker) http_mocker.get( _vendor_direct_fulfillment_shipping_request().build(), @@ -161,12 +155,11 @@ def test_given_http_status_500_on_availability_when_read_then_raise_system_error ) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE), expecting_exception=True) - assert output.errors[-1].trace.error.failure_type == FailureType.config_error + assert output.errors[-1].trace.error.failure_type == FailureType.system_error @freezegun.freeze_time(NOW.isoformat()) class TestIncremental: - @staticmethod def _read( config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False @@ -196,9 +189,10 @@ def test_when_read_then_state_message_produced_and_state_match_latest_record(sel mock_auth(http_mocker) http_mocker.get( _vendor_direct_fulfillment_shipping_request().build(), - _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).with_record( - _shipping_label_record() - ).build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), ) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) @@ -217,21 +211,21 @@ def test_given_state_when_read_then_state_value_is_created_after_query_param(sel _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), } - query_params_incremental_read = { - _REPLICATION_START_FIELD: state_value, _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT) - } + query_params_incremental_read = {_REPLICATION_START_FIELD: state_value, _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT)} http_mocker.get( _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_first_read).build(), - _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).with_record( - _shipping_label_record() - ).build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), ) http_mocker.get( _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_incremental_read).build(), - _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).with_record( - _shipping_label_record() - ).build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), ) output = self._read( diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py index 4c2327cdffea..9bdebc0ce6e1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py @@ -63,7 +63,6 @@ def _order_record() -> RecordBuilder: @freezegun.freeze_time(NOW.isoformat()) class TestFullRefresh: - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -76,9 +75,7 @@ def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> Entrypoi @HttpMocker() def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: mock_auth(http_mocker) - http_mocker.get( - _vendor_orders_request().build(), _vendor_orders_response().with_record(_order_record()).build() - ) + http_mocker.get(_vendor_orders_request().build(), _vendor_orders_response().with_record(_order_record()).build()) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) assert len(output.records) == 1 @@ -130,9 +127,7 @@ def test_given_two_slices_when_read_then_return_records(self, http_mocker: HttpM assert len(output.records) == 2 @HttpMocker() - def test_given_http_status_500_then_200_when_read_then_retry_and_return_records( - self, http_mocker: HttpMocker - ) -> None: + def test_given_http_status_500_then_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: mock_auth(http_mocker) http_mocker.get( _vendor_orders_request().build(), @@ -146,21 +141,16 @@ def test_given_http_status_500_then_200_when_read_then_retry_and_return_records( assert len(output.records) == 1 @HttpMocker() - def test_given_http_status_500_on_availability_when_read_then_raise_system_error( - self, http_mocker: HttpMocker - ) -> None: + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: mock_auth(http_mocker) - http_mocker.get( - _vendor_orders_request().build(), response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR) - ) + http_mocker.get(_vendor_orders_request().build(), response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE), expecting_exception=True) - assert output.errors[-1].trace.error.failure_type == FailureType.config_error + assert output.errors[-1].trace.error.failure_type == FailureType.system_error @freezegun.freeze_time(NOW.isoformat()) class TestIncremental: - @staticmethod def _read( config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False @@ -176,9 +166,7 @@ def _read( @HttpMocker() def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: mock_auth(http_mocker) - http_mocker.get( - _vendor_orders_request().build(), _vendor_orders_response().with_record(_order_record()).build() - ) + http_mocker.get(_vendor_orders_request().build(), _vendor_orders_response().with_record(_order_record()).build()) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) expected_cursor_value = _END_DATE.strftime(TIME_FORMAT) @@ -208,9 +196,7 @@ def test_given_state_when_read_then_state_value_is_created_after_query_param(sel _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), } - query_params_incremental_read = { - _REPLICATION_START_FIELD: state_value, _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT) - } + query_params_incremental_read = {_REPLICATION_START_FIELD: state_value, _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT)} http_mocker.get( _vendor_orders_request().with_query_params(query_params_first_read).build(), diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py index 58de04a90a9c..4631a8adfd48 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py @@ -67,11 +67,7 @@ def mock_auth(http_mocker: HttpMocker) -> None: http_mocker.post(RequestBuilder.auth_endpoint().build(), build_response(response_body, status_code=HTTPStatus.OK)) -def assert_message_in_log_output( - message: str, entrypoint_output: EntrypointOutput, log_level: Optional[Level] = Level.WARN -) -> None: +def assert_message_in_log_output(message: str, entrypoint_output: EntrypointOutput, log_level: Optional[Level] = Level.WARN) -> None: assert any( - message in airbyte_message.log.message - for airbyte_message in entrypoint_output.logs - if airbyte_message.log.level == log_level + message in airbyte_message.log.message for airbyte_message in entrypoint_output.logs if airbyte_message.log.level == log_level ) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py index 1421de0b1f3d..fec066fc19e8 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -34,6 +34,7 @@ def test_migrate_config(self, capsys): assert control_msg["type"] == Type.CONTROL.value assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value migrated_config = control_msg["control"]["connectorConfig"]["config"] + print(migrated_config) assert migrated_config["account_type"] == "Seller" def test_should_not_migrate(self): diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index 74b1806f1ada..c9f86dc90d29 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -9,7 +9,7 @@ import pytest from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner import SourceAmazonSellerPartner -from source_amazon_seller_partner.streams import VendorSalesReports +from source_amazon_seller_partner.streams import VendorOrders from source_amazon_seller_partner.utils import AmazonConfigException logger = logging.getLogger("airbyte") @@ -81,7 +81,7 @@ def test_check_connection_with_vendor_report(mocker, requests_mock, connector_ve json={"access_token": "access_token", "expires_in": "3600"}, ) - with patch.object(VendorSalesReports, "read_records", return_value=iter([{"some_key": "some_value"}])): + with patch.object(VendorOrders, "read_records", return_value=iter([{"some_key": "some_value"}])): assert SourceAmazonSellerPartner().check_connection(logger, connector_vendor_config_with_report_options) == (True, None) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py index 5a3a86188f53..a6e3661c5846 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -118,11 +118,12 @@ def test_read_records_retrieve_fatal(self, report_init_kwargs, mocker, requests_ status_code=201, json={"reportId": report_id}, ) + document_id = "some_document_id" requests_mock.register_uri( "GET", f"https://test.url/reports/2021-06-30/reports/{report_id}", status_code=200, - json={"processingStatus": ReportProcessingStatus.FATAL, "dataEndTime": "2022-10-03T00:00:00Z"}, + json={"processingStatus": ReportProcessingStatus.FATAL, "dataEndTime": "2022-10-03T00:00:00Z", "reportDocumentId": document_id}, ) stream = SomeReportStream(**report_init_kwargs) @@ -136,8 +137,8 @@ def test_read_records_retrieve_fatal(self, report_init_kwargs, mocker, requests_ ) ) assert e.value.internal_message == ( - f"Failed to retrieve the report 'GET_TEST_REPORT' for period {stream_start}-{stream_end} " - "due to Amazon Seller Partner platform issues. This will be read during the next sync." + f"Failed to retrieve the report 'GET_TEST_REPORT' for period {stream_start}-{stream_end}. " + "This will be read during the next sync. Error: Failed to retrieve the report result document." ) def test_read_records_retrieve_cancelled(self, report_init_kwargs, mocker, requests_mock, caplog): @@ -274,9 +275,7 @@ def test_stream_slices(self, report_init_kwargs, start_date, end_date, stream_st stream = VendorDirectFulfillmentShipping(**report_init_kwargs) with patch("pendulum.now", return_value=pendulum.parse("2022-09-05T00:00:00Z")): - assert list( - stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state=stream_state) - ) == expected_slices + assert list(stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state=stream_state)) == expected_slices @pytest.mark.parametrize( ("stream_slice", "next_page_token", "expected_params"), @@ -297,10 +296,8 @@ def test_stream_slices(self, report_init_kwargs, start_date, end_date, stream_st ), (None, {"nextToken": "123123123"}, {"nextToken": "123123123"}), (None, None, {}), - ) + ), ) def test_request_params(self, report_init_kwargs, stream_slice, next_page_token, expected_params): stream = VendorDirectFulfillmentShipping(**report_init_kwargs) - assert stream.request_params( - stream_state={}, stream_slice=stream_slice, next_page_token=next_page_token - ) == expected_params + assert stream.request_params(stream_state={}, stream_slice=stream_slice, next_page_token=next_page_token) == expected_params diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 5933cb0d5fb3..ed6daa83ca62 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -46,6 +46,8 @@ This page contains the setup guide and reference information for the Amazon Sell ## Step 2: Set up the source connector in Airbyte +To pass the check for Seller and Vendor accounts, you must have access to the [Orders endpoint](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference) and the [Vendor Orders endpoint](https://developer-docs.amazon.com/sp-api/docs/vendor-orders-api-v1-reference#get-vendorordersv1purchaseorders), respectively. + **For Airbyte Cloud:** From 3a82d0e5d15ff2297a8976bb9c48ae8bad99bec4 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Mon, 19 Feb 2024 17:26:56 +0200 Subject: [PATCH 7/7] Fix unit tests --- .../unit_tests/integration/test_report_based_streams.py | 4 ++-- .../integration/test_vendor_direct_fulfillment_shipping.py | 2 +- .../unit_tests/integration/test_vendor_orders.py | 2 +- .../unit_tests/test_migrations.py | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py index 5c71bd43b428..bfdbfa241e73 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py @@ -388,10 +388,10 @@ def test_given_report_status_fatal_when_read_then_exception_raised( ) output = self._read(stream_name, config(), expecting_exception=True) - assert output.errors[-1].trace.error.failure_type == FailureType.system_error + assert output.errors[-1].trace.error.failure_type == FailureType.config_error assert ( f"Failed to retrieve the report '{stream_name}' for period {CONFIG_START_DATE}-{CONFIG_END_DATE}. This will be read during the next sync. Error: {{'errorDetails': 'Error in report request: This report type requires the reportPeriod, distributorView, sellingProgram reportOption to be specified. Please review the document for this report type on GitHub, provide a value for this reportOption in your request, and try again.'}}" - ) in output.errors[-1].trace.error.internal_message + ) in output.errors[-1].trace.error.message @pytest.mark.parametrize( ("stream_name", "date_field", "expected_date_value"), diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py index c9755255b550..7706c715d3c4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py @@ -155,7 +155,7 @@ def test_given_http_status_500_on_availability_when_read_then_raise_system_error ) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE), expecting_exception=True) - assert output.errors[-1].trace.error.failure_type == FailureType.system_error + assert output.errors[-1].trace.error.failure_type == FailureType.config_error @freezegun.freeze_time(NOW.isoformat()) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py index 9bdebc0ce6e1..691bd32608de 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py @@ -146,7 +146,7 @@ def test_given_http_status_500_on_availability_when_read_then_raise_system_error http_mocker.get(_vendor_orders_request().build(), response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)) output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE), expecting_exception=True) - assert output.errors[-1].trace.error.failure_type == FailureType.system_error + assert output.errors[-1].trace.error.failure_type == FailureType.config_error @freezegun.freeze_time(NOW.isoformat()) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py index fec066fc19e8..1421de0b1f3d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -34,7 +34,6 @@ def test_migrate_config(self, capsys): assert control_msg["type"] == Type.CONTROL.value assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value migrated_config = control_msg["control"]["connectorConfig"]["config"] - print(migrated_config) assert migrated_config["account_type"] == "Seller" def test_should_not_migrate(self):