diff --git a/.gitignore b/.gitignore index 50f4345..1f3f0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ singer-check-tap-data dist/ __pycache__/ .idea -.secrets \ No newline at end of file +.secrets +.venv \ No newline at end of file diff --git a/tap_chargebee/streams/base.py b/tap_chargebee/streams/base.py index fc48bb1..cf4a1e0 100644 --- a/tap_chargebee/streams/base.py +++ b/tap_chargebee/streams/base.py @@ -208,7 +208,7 @@ def sync_data(self): params[field_name] = field_value LOGGER.info("Querying filtering by {}={}".format(field_name, field_value)) - ids = [] + ids = dict() while not done: max_date = bookmark_date @@ -243,15 +243,15 @@ def sync_data(self): if self.ENTITY == 'coupon': for coupon in Util.coupons: to_write.append(coupon) - if self.ENTITY == 'transaction': - # store ids to clean dupplicates - to_write = [record for record in to_write if record["id"] not in ids] - ids.extend([trans["id"] for trans in to_write]) + if self.ENTITY in ['transaction', 'subscription']: + # store ids to clean dupplicates, keep only the last appearance of a record id + for record in to_write: + ids[record["id"]] = record with singer.metrics.record_counter(endpoint=table) as ctr: - singer.write_records(table, to_write) - - ctr.increment(amount=len(to_write)) + if not ids: + singer.write_records(table, to_write) + ctr.increment(amount=len(to_write)) if bookmark_key is not None: for item in to_write: @@ -268,7 +268,7 @@ def sync_data(self): datetime.fromtimestamp(item.get(bookmark_key), tz=dtz.gettz('UTC') )) - if bookmark_key is not None: + if bookmark_key is not None and not ids: self.state = incorporate( self.state, table, 'bookmark_date', max_date) @@ -288,6 +288,16 @@ def sync_data(self): LOGGER.info(f"Advancing by one offset [{params}]") save_state(self.state) + if ids: + with singer.metrics.record_counter(endpoint=table) as ctr: + to_write = list(ids.values()) + singer.write_records(table, to_write) + ctr.increment(amount=len(to_write)) + if bookmark_key is not None: + self.state = incorporate( + self.state, table, 'bookmark_date', max_date) + save_state(self.state) + def sync_parent_data(self): table = self.TABLE