From de02287e8215397be2513f995b915319e7e655d0 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 2 Aug 2023 10:11:42 -0400 Subject: [PATCH 1/3] TDL-23384: Add retry for chunked encoding errors (#61) * add retry for chunked encoding error * Add exponential backoff to `request_export` * Remove backoff because this is a generator Throwing an exception in the generator seems to just raise a `StopIteration` * Add retry logic to the caller of `client.request_export` * Add a test for retrying on `ChunkedEncodingError`s * Bump to `v1.5.1`, update changelog --------- Co-authored-by: Leslie VanDeMark --- CHANGELOG.md | 3 ++ setup.py | 2 +- tap_mixpanel/client.py | 4 +-- tap_mixpanel/streams.py | 9 ++++++ tests/unittests/test_error_handling.py | 39 ++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fe1ef4..101d9d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.5.1 + * Add retry logic for `ChunkedEncodingError`s [#61](https://github.com/singer-io/tap-mixpanel/pull/61) + ## 1.5.0 * Adds `export_events` as optional param to filter the data for export stream based on event names [#56](https://github.com/singer-io/tap-mixpanel/pull/56) diff --git a/setup.py b/setup.py index 57abc3e..f6c4dee 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup(name='tap-mixpanel', - version='1.5.0', + version='1.5.1', description='Singer.io tap for extracting data from the mixpanel API', author='jeff.huth@bytecode.io', classifiers=['Programming Language :: Python :: 3 :: Only'], diff --git a/tap_mixpanel/client.py b/tap_mixpanel/client.py index 544f3d4..4e49a5d 100644 --- a/tap_mixpanel/client.py +++ b/tap_mixpanel/client.py @@ -4,7 +4,7 @@ import jsonlines import requests import singer -from requests.exceptions import ConnectionError, Timeout +from requests.exceptions import ChunkedEncodingError, ConnectionError, Timeout from requests.models import ProtocolError from singer import metrics @@ -204,7 +204,7 @@ def check_access(self): @backoff.on_exception( backoff.expo, - (Server5xxError, Server429Error, ReadTimeoutError, ConnectionError, Timeout, ProtocolError), + (Server5xxError, Server429Error, ReadTimeoutError, ConnectionError, Timeout, ProtocolError, ChunkedEncodingError), max_tries=BACKOFF_MAX_TRIES_REQUEST, factor=3, logger=LOGGER, diff --git a/tap_mixpanel/streams.py b/tap_mixpanel/streams.py index d8c00b9..da89c9e 100644 --- a/tap_mixpanel/streams.py +++ b/tap_mixpanel/streams.py @@ -7,6 +7,8 @@ import urllib import pytz +import requests +import backoff import singer from singer import Transformer, metadata, metrics, utils from singer.utils import strptime_to_utc @@ -695,6 +697,13 @@ class Export(MixPanel): replication_method = "INCREMENTAL" params = {} + + @backoff.on_exception( + backoff.expo, + (requests.exceptions.ChunkedEncodingError,), + max_tries=5, + factor=2, + ) def get_and_transform_records( self, querystring, diff --git a/tests/unittests/test_error_handling.py b/tests/unittests/test_error_handling.py index 7ba6abc..0b973c8 100644 --- a/tests/unittests/test_error_handling.py +++ b/tests/unittests/test_error_handling.py @@ -1,10 +1,12 @@ import unittest import requests +import jsonlines from unittest import mock from parameterized import parameterized from tap_mixpanel import client +from tap_mixpanel import streams # Mock response REQUEST_TIMEOUT = 300 @@ -249,3 +251,40 @@ def test_check_access_handle_timeout_error(self, mock_request, mock_time): # Verify that requests.Session.request is called 5 times self.assertEqual(mock_request.call_count, 5) + + @mock.patch("jsonlines.jsonlines.Reader.iter", side_effect=requests.exceptions.ChunkedEncodingError) + def test_ChunkedEncodingError(self, mock_jsonlines, mock_time): + """ + Check whether the request backoffs properly for `check_access` method for 5 times in case of Timeout error. + """ + mock_client = client.MixpanelClient(api_secret="mock_api_secret", api_domain="mock_api_domain", request_timeout=REQUEST_TIMEOUT) + mock_client._MixpanelClient__verified = True + + fake_response = MockResponse(500) + fake_response.iter_lines = lambda : [] + mock_client.perform_request = lambda *args, **kwargs: fake_response + + stream = streams.Export(mock_client) + + with self.assertRaises(requests.exceptions.ChunkedEncodingError) as error: + stream.get_and_transform_records( + querystring={}, + project_timezone=None, + max_bookmark_value=None, + state=None, + config=None, + catalog=None, + selected_streams=None, + last_datetime=None, + endpoint_total=None, + limit=None, + total_records=None, + parent_total=None, + record_count=None, + page=None, + offset=None, + parent_record=None, + date_total=None, + ) + + self.assertEqual(mock_jsonlines.call_count, 5) From d3f3430457347d78a2ce9c12c6afe91548fb38f8 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark <38043390+leslievandemark@users.noreply.github.com> Date: Thu, 18 Jan 2024 13:26:31 -0500 Subject: [PATCH 2/3] Upgrade python version to 3.11.7 (#63) * Upgrade pip version to be compatible with python 3.11 * try nose2 on python 3.11.7 * use correct tests directory * try install parameteriZed * bump backoff version * bump singer-python version for new backoff version * run tests on 3.11 * add coverage back in * version bump and changelog update --- .circleci/config.yml | 9 +++++---- CHANGELOG.md | 3 +++ setup.py | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e45b7bd..e06e7a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2 jobs: build: docker: - - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester + - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester-3-11-dev steps: - checkout - add_ssh_keys @@ -14,7 +14,7 @@ jobs: source dev_env.sh python3 -m venv /usr/local/share/virtualenvs/tap-mixpanel source /usr/local/share/virtualenvs/tap-mixpanel/bin/activate - pip install -U 'pip<19.2' 'setuptools<51.0.0' + pip install -U 'pip==23.3.2' 'setuptools<51.0.0' pip install .[dev] pip install pytest-cov - run: @@ -33,8 +33,9 @@ jobs: name: 'Unit Tests' command: | source /usr/local/share/virtualenvs/tap-mixpanel/bin/activate - pip install nose coverage parameterized - nosetests --with-coverage --cover-erase --cover-package=tap_mixpanel --cover-html-dir=htmlcov tests/unittests + pip install nose2 parameterized nose2[coverage_plugin]>=0.6.5 + pip install parameterized + nose2 --with-coverage -v -s tests/unittests - store_test_results: path: test_output/report.xml - store_artifacts: diff --git a/CHANGELOG.md b/CHANGELOG.md index 101d9d2..be1b46e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.6.0 + * Requirement updates to run on python 3.11.7 [#63](https://github.com/singer-io/tap-mixpanel/pull/63) + ## 1.5.1 * Add retry logic for `ChunkedEncodingError`s [#61](https://github.com/singer-io/tap-mixpanel/pull/61) diff --git a/setup.py b/setup.py index f6c4dee..2889564 100644 --- a/setup.py +++ b/setup.py @@ -3,15 +3,15 @@ from setuptools import setup, find_packages setup(name='tap-mixpanel', - version='1.5.1', + version='1.6.0', description='Singer.io tap for extracting data from the mixpanel API', author='jeff.huth@bytecode.io', classifiers=['Programming Language :: Python :: 3 :: Only'], py_modules=['tap_mixpanel'], install_requires=[ - 'backoff==1.8.0', + 'backoff==2.2.1', 'requests==2.22.0', - 'singer-python==5.12.1', + 'singer-python==6.0.0', 'jsonlines==1.2.0' ], entry_points=''' From 63fcc1e2a2192a005eb4a2511deb6db265d7f71a Mon Sep 17 00:00:00 2001 From: Eivin Giske Skaaren Date: Tue, 3 Sep 2024 12:46:31 +0200 Subject: [PATCH 3/3] Enable copilot usage in PR template according to Qlik policy --- .github/pull_request_template.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6e46b00..ef49bc0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,3 +9,7 @@ # Rollback steps - revert this branch + +#### AI generated code +https://internal.qlik.dev/general/ways-of-working/code-reviews/#guidelines-for-ai-generated-code +- [ ] this PR has been written with the help of GitHub Copilot or another generative AI tool