✨ Source Tiktok Marketing: Support concurrency (#51584)

Co-authored-by: Anatolii Yatsuk <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Jan 31, 2025 · e91b74d · e91b74d
1 parent 9130c9f
commit e91b74d
Show file tree

Hide file tree

Showing 16 changed files with 1,387 additions and 983 deletions.
diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml
@@ -15,12 +15,6 @@ acceptance_tests:
     tests:
       - config_path: "secrets/prod_config.json"
         status: "succeed"
-      - config_path: "secrets/prod_config_with_day_granularity.json"
-        status: "succeed"
-      - config_path: "secrets/prod_config_with_lifetime_granularity.json"
-        status: "succeed"
-      - config_path: "secrets/new_config_prod.json"
-        status: "succeed"
       - config_path: "secrets/config_oauth.json"
         status: "succeed"
       - config_path: "integration_tests/invalid_config.json"
@@ -32,10 +26,7 @@ acceptance_tests:
   discovery:
     tests:
       - config_path: "secrets/prod_config.json"
-      - config_path: "secrets/prod_config_with_day_granularity.json"
-      - config_path: "secrets/prod_config_with_lifetime_granularity.json"
       - config_path: "secrets/config.json"
-      - config_path: "secrets/new_config_prod.json"
       - config_path: "secrets/config_oauth.json"
   basic_read:
     tests:
@@ -59,31 +50,8 @@ acceptance_tests:
             bypass_reason: "No data in the integration test account. We should seed the sandbox later on."
           - name: creative_assets_music
             bypass_reason: "System music provided by TikTok - very volatile data."
-        # Old style streams with granularity config (for < 0.1.13)
-        # Note: not needed to be tested separately in full and incremental tests, because code of
-        # these streams is called directly in new style streams
-      - config_path: "secrets/prod_config_with_day_granularity.json"
-        expect_records:
-          path: "integration_tests/expected_records2.jsonl"
-          exact_order: no
-        timeout_seconds: 1200
-        empty_streams:
-          - name: ad_groups
-            bypass_reason: "This stream is tested on the new style config."
-          - name: ads
-            bypass_reason: "This stream is tested on the new style config."
-          - name: advertiser_ids
-            bypass_reason: "This stream is tested on the new style config."
-          - name: advertisers
-            bypass_reason: "This stream is tested on the new style config."
-          - name: audiences
-            bypass_reason: "No data in the integration test account. We should seed the sandbox later on."
-          - name: campaigns
-            bypass_reason: "This stream is tested on the new style config."
-          - name: creative_assets_portfolios
+          - name: advertisers_audience_reports_lifetime
             bypass_reason: "No data in the integration test account. We should seed the sandbox later on."
-          - name: creative_assets_music
-            bypass_reason: "System music provided by TikTok - very volatile data."
   full_refresh:
     tests:
       - config_path: "secrets/config.json"
@@ -110,4 +78,4 @@ acceptance_tests:
         timeout_seconds: 7200
         skip_comprehensive_incremental_tests: true
         future_state:
-          future_state_path: "integration_tests/abnormal_state.json"
+          bypass_reason: "This test does not make sense using Concurrent CDK"
diff --git a/...-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl b/...-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl
@@ -55,9 +55,6 @@
 {"stream": "advertisers_audience_reports_daily", "data": {"metrics": {"cpc": "0.38", "clicks": "6", "spend": "2.26", "cpm": "6.75", "ctr": "1.79", "impressions": "335"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_18_24", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1698064951205}
 {"stream": "advertisers_audience_reports_daily", "data": {"metrics": {"cpc": "0.00", "clicks": "1", "spend": "0.00", "cpm": "0.00", "ctr": "2.86", "impressions": "35"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_35_44", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698064951208}
 {"stream": "advertisers_audience_reports_daily", "data": {"metrics": {"cpc": "0.29", "clicks": "29", "spend": "8.32", "cpm": "3.88", "ctr": "1.35", "impressions": "2146"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "FEMALE", "stat_time_day": "2021-10-19 00:00:00"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1698064951211}
-{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.58", "cpc": "0.36", "spend": "31.56", "ctr": "1.26", "clicks": "87", "impressions": "6897"}, "dimensions": {"age": "AGE_35_44", "advertiser_id": 7002238017842757633, "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698064974164}
-{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "0.00", "cpc": "0.00", "spend": "0.00", "ctr": "0.00", "clicks": "0", "impressions": "17"}, "dimensions": {"age": "AGE_55_100", "advertiser_id": 7002238017842757633, "gender": "FEMALE"}, "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_55_100"}, "emitted_at": 1698064974168}
-{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "3.92", "cpc": "0.32", "spend": "7.13", "ctr": "1.21", "clicks": "22", "impressions": "1818"}, "dimensions": {"age": "AGE_13_17", "advertiser_id": 7002238017842757633, "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1698064974171}
 {"stream": "ad_group_audience_reports_by_country_daily", "data": {"metrics": {"campaign_id": 1714073078669329, "clicks": "65", "conversion": "0", "result": "65", "impressions": "4874", "cpc": "0.31", "real_time_result": "65", "real_time_conversion_rate": "0.00", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "dpa_target_audience_type": null, "spend": "20.00", "cost_per_result": "0.308", "real_time_cost_per_result": "0.308", "cost_per_conversion": "0.00", "tt_app_name": "0", "cpm": "4.10", "result_rate": "1.33", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "mobile_app_id": "0", "campaign_name": "Website Traffic20211019110444", "tt_app_id": "0", "promotion_type": "Website", "real_time_result_rate": "1.33", "ctr": "1.33", "adgroup_name": "Ad Group20211019111040"}, "dimensions": {"stat_time_day": "2021-10-19 00:00:00", "country_code": "US", "adgroup_id": 1714073022392322}, "stat_time_day": "2021-10-19 00:00:00", "adgroup_id": 1714073022392322, "country_code": "US"}, "emitted_at": 1698913315398}
 {"stream": "ad_group_audience_reports_by_country_daily", "data": {"metrics": {"campaign_id": 1714125042508817, "clicks": "69", "conversion": "0", "result": "69", "impressions": "5830", "cpc": "0.29", "real_time_result": "69", "real_time_conversion_rate": "0.00", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "dpa_target_audience_type": null, "spend": "20.00", "cost_per_result": "0.290", "real_time_cost_per_result": "0.290", "cost_per_conversion": "0.00", "tt_app_name": "0", "cpm": "3.43", "result_rate": "1.18", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "mobile_app_id": "0", "campaign_name": "Website Traffic20211020010104", "tt_app_id": "0", "promotion_type": "Website", "real_time_result_rate": "1.18", "ctr": "1.18", "adgroup_name": "Ad Group20211020010107"}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "country_code": "US", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106, "country_code": "US"}, "emitted_at": 1698913315402}
 {"stream": "ad_group_audience_reports_by_country_daily", "data": {"metrics": {"campaign_id": 1714125042508817, "clicks": "53", "conversion": "0", "result": "53", "impressions": "3765", "cpc": "0.38", "real_time_result": "53", "real_time_conversion_rate": "0.00", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "dpa_target_audience_type": null, "spend": "20.00", "cost_per_result": "0.377", "real_time_cost_per_result": "0.377", "cost_per_conversion": "0.00", "tt_app_name": "0", "cpm": "5.31", "result_rate": "1.41", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "mobile_app_id": "0", "campaign_name": "Website Traffic20211020010104", "tt_app_id": "0", "promotion_type": "Website", "real_time_result_rate": "1.41", "ctr": "1.41", "adgroup_name": "Ad Group20211020010107"}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "country_code": "US", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106, "country_code": "US"}, "emitted_at": 1698913315406}

diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml
@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
-  dockerImageTag: 4.3.7
+  dockerImageTag: 4.4.0-rc1
   dockerRepository: airbyte/source-tiktok-marketing
   documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing
   githubIssueLabel: source-tiktok-marketing
@@ -25,6 +25,8 @@ data:
     oss:
       enabled: true
   releases:
+    rolloutConfiguration:
+      enableProgressiveRollout: true
     breakingChanges:
       4.0.0:
         message: