From 7e4d3f1a47f59370f5a3fe1a0d3602203b55cca4 Mon Sep 17 00:00:00 2001 From: fsponga Date: Sun, 5 Mar 2023 19:21:14 +0100 Subject: [PATCH] Update airbnb_scraper.py I've updated the previous pull request to avoid an empty api request, this should fix the while loop --- stl/scraper/airbnb_scraper.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/stl/scraper/airbnb_scraper.py b/stl/scraper/airbnb_scraper.py index c7d044d..1f96333 100644 --- a/stl/scraper/airbnb_scraper.py +++ b/stl/scraper/airbnb_scraper.py @@ -39,9 +39,10 @@ def run(self, query: str, params: dict): pagination['totalCount'], self.__geography['fullAddress'], params) ) n_listings = 0 + max_pages = 1 page = 1 data_cache = {} - while pagination.get('hasNextPage'): + while page <= max_pages: self.__logger.info('Searching page {} for {}'.format(page, query)) listing_ids = self.__pdp.collect_listings_from_sections(data, self.__geography, data_cache) for listing_id in listing_ids: # request each property page @@ -67,11 +68,13 @@ def run(self, query: str, params: dict): self.__logger.info(msg) listings.append(listing) - self.__add_search_params(params, url) - items_offset = pagination['itemsOffset'] - params.update({'itemsOffset': items_offset}) - url = self.__explore.get_url(query, params) - data, pagination = self.__explore.search(url) + if pagination.get('hasNextPage'): + self.__add_search_params(params, url) + items_offset = pagination['itemsOffset'] + params.update({'itemsOffset': items_offset}) + url = self.__explore.get_url(query, params) + data, pagination = self.__explore.search(url) + max_pages+=1 page += 1 self.__persistence.save(query, listings)